#!/usr/bin/env perl

# 
# Copyright (c) 2002-2007 Eric Wallengren
# This file is part of the Continuous Automated Build and Integration 
# Environment (CABIE)
# 
# CABIE is distributed under the terms of the GNU General Public
# License version 2 or any later version.  See the file COPYING for copying 
# permission or http://www.gnu.org. 
#                                                                            
# THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED OR  
# IMPLIED, without even the implied warranty of MERCHANTABILITY or FITNESS 
# FOR A PARTICULAR PURPOSE.  ANY USE IS AT YOUR OWN RISK. 
#                                                                            
# Permission to modify the code and to distribute modified code is granted, 
# provided the above notices are retained, and a notice that the code was 
# modified is included with the above copyright notice. 
# 

use strict;
no  strict "refs";

BEGIN{push @INC, "lib";}

#
# The buildserver needs to be handled differently
# on a non-posix system (Windows)
#
my $POSIX = 1;

my $ospackage;

my $rpcserver;

#
# Make this work on my Mac...
#
my $bamDarwin=0;;

#
# See if this is a windoz system...
#
if ($ =~ /MSWin32/) {
    $POSIX = 0;
    $ospackage  = "winsys";
} else {
    $ospackage  = "unixsys";
    if ($ =~ /darwin/ic) {
        $bamDarwin = 1;
    }
}

use Getopt::Std;

my %Opts;

use Socket;
use FindBin;
use Cwd;
use threads;

my $mthread;
my $pthread;
my $svcthread;
my $dir;
my $cmdkey;

$dir = cwd();

use Sys::Hostname;
use File::Copy;
use File::Path;
use IO::Handle;
use Mail::Sendmail;
use Socket;

use RPC::XML;
use RPC::XML::Server;
use RPC::XML::Client;
use common;
use htmlog;

#
# Use package for all SQL queries...
#
use queries;

#
# Use package for supported command set
#
use cmdset;

my $suppcmds = new cmdset;
my %suppcmds = %{ $suppcmds };

my $qs = new queries;

#
# Use signal handler to trap signal errors
#
# use sigtrap qw (handler _catch_sig normal-signals stack-trace error-signals);

#
# We don't care about child processes.
#
$SIG{CHLD} = 'IGNORE';

my $hostname     = hostname();
$hostname        =~ s/\.[a-zA-Z0-9]+//g;

my $configname   = $hostname;
$configname      =~ s/-//g;

#
# Load master configuration file
#
require "$configname.pm";

#
# Create instances of packages enabled above...
#
my $config = new $configname;

#
# unixsys for posix
# winsys for windows
#
require "$ospackage.pm";

#
# Create instances of packages enabled above...
# unixsys for posix
# winsys for windows
#
my $os  = new $ospackage;

#
# Change directories to where the buildserver lives
#
chdir $config->BSR;

$dir = cwd();

#
# Use this for overriding default environment variables
#
if (-f "$dir/config/vars.conf") {
    open (VARIN, "<$dir/config/vars.conf");
    open (VAROUT, ">vars.set");
    while (<VARIN>) {
        chomp;
        my ($envname, $envvalue) = split(/=/, $_);
        $ENV{"$envname"} = "$envvalue";
        print VAROUT "$envname = $envvalue\n";
    } 
    close (VAROUT);
    close(VARIN);
}

open (DEBUGGER, ">$dir/logs/dbg.log");
DEBUGGER->autoflush(1);

unshift (@INC, "lib"); # Prepend a directory name

#
# Some boolean globals for forking...
#
my ($bMonitor, $bServer, $bDaemon);

#
# Process options...
#
if ($POSIX) {
    getopts('s:m:dvt', \%Opts);
} else {
    getopts('s:m:irx', \%Opts);
}

my $hostname     = hostname();
$hostname        =~ s/\.[a-zA-Z0-9]+//g;

my $port         = $os->port;
my $sport        = $os->sport;
my $mport        = $os->mport;
my $makedaemon   = $os->makedaemon;
my $makemonitor  = $os->makemonitor;
my $logext       = "connectlog";
my $errlogext    = "errorlog";
my $host         = "$hostname";

my @addresses = gethostbyname($hostname);
@addresses = map { inet_ntoa($_) } @addresses[4 .. $#addresses];

# $hostname = gethostbyaddr(inet_aton($addresses[0]), AF_INET) 
#    || die "Can't resolve $addresses[0]: $!\n";


#
# Use generic routine from OS package to validate args passed
# to the buildserver...
#
$os->validateargs($hostname, %Opts);

#
# Name of this buildserver, and boolean value for authorization
#
my $name       = "$hostname";

my $authorized = 0;
#
# Globally declared sccs
#
my $globsccs;

#
# Use for supported SCCS systems...
#
my $bIsPerforce = 0;
my $bIsCvs      = 0;

#
# Call CM broker
#
use cmbroker;
my $cmbroker = new cmbroker;

#
# Start global declarations
#
my $kidpid;
my $item;
my $peeraddress;
my $peername;
my $reqtime;
my $isinternal = 0;
my $isexternal = 0;
my $bBuild     = 0;

my @myout;
my @cmdarray;

#
# Primarily unused
#
my $CVSROOT    = $config->CVSROOT;

#
# Use for keep and free commands
#
# OS SPECIFIC
my $Moutdir    = $config->JOBDIR;
my $Mtmpdir    = $config->BTMP;
my $Mnull      = $config->NULL;

#
# Icons...
#
my $disicon    = $config->DISICON;
my $cgibin     = $config->CGIBIN;
my $logo       = $config->LOGOICON;
my $greenicon  = $config->GREENICON;
my $redicon    = $config->REDICON;
my $yellowicon = $config->YELLOWICON;
my $runicon    = $config->RUNICON;
my $openicon   = $config->OPENFOLD;
my $closedicon = $config->CLOSEDFOLD;
my $promoicon  = $config->PROMOICON;
my $webserver  = $config->WEBSERVER;
my $tested     = $config->TESTED;
my $nottested  = $config->NOTTESTED;

#
# Perforce user
#
my $p4user     = $config->P4USER;

#
# Vars for genweb...
#
my @PLATFORM = (
    "$hostname"
);

#
# More globals, these are for the generation of the web display
#
my %mainhash;
my %jobhash;
my @jobarray;
my @stats;
my %idlehash;
my %statehash;
my @reversearray;
my @timearray;
my %workhash;
my $viewitems;
my $HASARGS;
my $itsnow;
my @webarray;
my %jobHash;
my %processtree;
my %foundhash;

# Found by strict 
my @JobList;

#
# Colors used for genweb
#
my @colorarray = (
    "#33cc99",
    "#dc1973",
    "#fe5066",
    "#b9ffb9",
    "#ffcc00",
    "#bed2ff",
    "#669999",
    "#ff9248"
);

#
# Error messages...
#
my %msghash = (
    0,  "Command request accepted",
    1,  "Previous job running",
    2,  "Job undefined",
    3,  "Unsupported request",
    4,  "Help request",
    5,  "locked job",
    6,  "unable to change status",
    7,  "invalid arguments",
    8,  "cat what?",
    9,  "incomplete request",
    10, "build what?",
    11, "\(command not yet supported\)",
    12, "entry already exists",
    13, "entry does not exist",
    14, "no jobs defined on server",
    15, "status change pending",
    16, "job has never been defined",
    17, "job is already defined",
    18, "build not found",
    19, "build already promoted",
    20, "failed to make directory",
    21, "out of disk space",
    22, "no such server",
    23, "command not supported",
    24, "missing server name",
    25, "missing configuration file",
);

#
# Commands used by both the user and admin modes
#
my %sharedcmds   = %{ $suppcmds{sharedcmds} };
my %smadmincmds  = %{ $suppcmds{smadmincmds} };
my %smsharedcmds = %{ $suppcmds{smsharedcmds} };
my %usercmds     = %{ $suppcmds{usercmds} }; 
my %admincmds    = %{ $suppcmds{admincmds} };
my %signatures   = %{ $suppcmds{signatures} };

#
# Declare hash for commands
#
my %supportedcmds;

#
# Declare hash table for help
#
my %helptable;

#
# Start time of the build server
#
my $serverstarttime = sprintf "%s", scalar localtime;

$os->startdaemons($dir, %Opts);

chdir $config->BSR;

#
# IMPORTANT:
#
# The server will fail if Perl doesn't support threads.  The lines 
# beginning with mthread and pthread can be commented out here if there's
# no thread support but means that the server won't poll for changes.
#
if ($os->bAmMonitor) {
    %sharedcmds      = %smsharedcmds;
    %admincmds       = %smadmincmds;

    #
    # We can call the function directly from threads under
    # unix otherwise we pass a reference
    #
    if ($POSIX) {
        $mthread     = threads->new("polljobs");
    } else {
        $mthread     = threads->new(\&polljobs);
        $svcthread   = threads->new($os->_monitor);
    }
    $logext          = "_connectlog";
    $errlogext       = "_errorlog";
    $sport           = $os->sport;
} else {
    if ($POSIX) {
        $pthread     = threads->new("pendingjobs");
    } else {
        $pthread     = threads->new(\&pendingjobs);
	# $svcthread   = threads->new($os->_monitor);
    }
}

$rpcserver=RPC::XML::Server->new(host=>$host, port=>$os->port);

if (!$os->makedaemon) {
    $port = $sport;
    print "***$port non-daemon starting server***\n";
    $rpcserver=RPC::XML::Server->new(host=>$host, port=>$sport);
}

foreach $cmdkey (keys %signatures) {

    $signatures{$cmdkey}{code}=\&cmd_broker;

    $rpcserver->add_proc($signatures{$cmdkey});
}

#
# Flush any data written to STDIO
#
STDOUT->autoflush(1);
STDERR->autoflush(1);

#
# Add entry about server in SQL 
#
if ($os->bAmServer) {
    _registerserver();
}

#
# Get the current time
#
my $sreqtime = scalar localtime;

#
# Log buildserver start
#
_logevents("[$sreqtime $name]: buildserver connectlog session started", 0);
_logevents("[$sreqtime $name]: buildserver rejectlog session started", 1);

# Print to the console the server status...
print "[$sreqtime $name]: waiting for events\n";

# Wait for an incoming request...
if ($os->bAmServer) {
    $rpcserver->server_loop();
} else {
    $rpcserver->server_loop();
}

# Here's the login function, we'll look for command authorization...
#-------------------------------------------------------------------
sub login_proc {

    #
    # For use with mysql
    #
    my $sqlquery;
    my @sqlarray;
    my $line;

    # Get the connection handle...
    # my ($conn) = shift;

    # Assume that there's no build authorization...
    $authorized = 1;

    # Get the socket from the connection handle...
    #my $socket = $conn->{sock};

    # Get the client name from the socket...
    #my $other_end = getpeername($socket);

    # Get the ip address for the host...
    #my ($port, $ipaddr) = unpack_sockaddr_in($other_end);
    #my $actual_ip = inet_ntoa($ipaddr);   

    $os->tracefunction((caller(0))[3]);

    # Get the host name...
    #$peername = gethostbyaddr($ipaddr, AF_INET);

    #if (!defined($peername)) {
    #    $peername = $actual_ip;
    #}

    # Use $peeraddress for console messages...
    #$peeraddress  = "$peername"." \($actual_ip\)]";

    #
    # SQL way to determine authorization...
    #
    #$sqlquery = $qs->formatquery(45, $hostname);

    #@sqlarray = $os->run_sql_query("$sqlquery", ",", 0);
    
    #if (@sqlarray == 0) {
    #    $authorized = 1;
    #} else {
    #    foreach $line (@sqlarray) {
    #        chomp $line;
    #        if ($peername =~ /^$line$/i) {
    #            $authorized = 1;
    #        }
    #    }
    #}

    if ($authorized) {
        %supportedcmds = (%sharedcmds, %admincmds);
    } else {
        %supportedcmds = (%sharedcmds, %usercmds);
    }

    $reqtime = sprintf "[%s", scalar localtime;

}

sub cmd_broker {

    # my $conn = shift;

    # login_proc($conn);
    login_proc();

    my %rpchash = %{ $rpcserver };

    unshift (@_, $rpchash{method_name}." ");

    @ARGV = @_;

    my $msg  = join(/ /, @ARGV);

    # Variables to be used by this subroutine...
    my $buildreq;
    my $ServerName;
    my $cmd;
    my $ret;

    my $sqlquery;
    my @sqlarray;

    my @JOBArray;
    my @returnarray;

    @cmdarray = ();

    $os->tracefunction((caller(0))[3]);

    #
    # Was there a message from the client
    #
    if (defined $msg && $msg !~ /^$/) {

        # Strip newline
        chomp $msg;

        @cmdarray = split(/ /,$msg);

        # Create cmd array
        $cmd = shift @cmdarray;

        if ($cmd =~ /^helo$/) {
            return ["helo from $hostname"];
        }

        # Search hash for supported request
        my $cmdtype = $supportedcmds{$cmd};

        if (!defined($cmdtype)) {
            return [$msghash{3}];
            _logevents("\"$msg\": $msghash{3}",1);
            return [""];
        }

        #
        # Determine internal/external function.
        #
        my $int_ext  = $supportedcmds{$cmd}{type};
        my $function = $supportedcmds{$cmd}{function};

        # If cmd is valid
        if (defined($cmd)) {

            # We need to determine if this is a build command, special 
            # handling needs to take place if it is...
            if ($cmd =~ /^build$/) {

                my %Buildopts;

                my @newarray = _parsearray(@cmdarray);

                my $funcstring;

                $funcstring = sprintf("$function", $dir);

                print STDERR "$funcstring\n";

                @ARGV = @newarray;

                getopts('n:j:S:', \%Buildopts);

                $buildreq   = $Buildopts{n};
                $ServerName = $Buildopts{S};

                #
                # See if servername is defined
                #
                if (!defined($ServerName)) {
                    $ServerName = $hostname;
                }

                if ($ServerName =~ /^$hostname$/) {

                    #
                    # This is a build...
                    #
                    my $locked;
    
                    $bBuild = 1;
               
                    #
                    # Make sure a jobname was sent
                    #
                    if (!defined($buildreq)) {
                        return[(display_command_usage("build"))];
                        _logevents("$msg: $msghash{9}",1);
                    }
    
                    #
                    # Make sure it's a valid active job
                    #
                    if (!_ValidJob($buildreq,$hostname,0)) {
                        return ["undefined job $buildreq"];
                        _logevents("$cmdarray[0] $buildreq: $msghash{2}",1);
                        return [""];
                    }
    
                    #
                    # SQL query to see if it's in a locked state
                    #
                    $sqlquery = $qs->formatquery(51,$buildreq,$hostname,1); 

                    @sqlarray = $os->run_sql_query($sqlquery, ";");
    
                    #
                    # See if sql returned a record and return it to
                    # the client.
                    #
                    $locked = @sqlarray;
                    if ($locked) {
                        return ["$sqlarray[0]"];
                        _logevents("$cmd $buildreq $hostname: $sqlarray[0]",1);
                        return [""];
                    }
    
                    #
                    # Check for free space...
                    #
                    if ((_freespace($buildreq, 0)) == 1) {
                        return[$msghash{21}];
                        # Display a message to stdout
                        _logevents("$msg: $msghash{21}",1);
                        return [""];
                    }
    
                    #
                    # Call the build.
                    #
                    external($funcstring, @newarray);
                    return[$msghash{0}];
                    # Display a message to the console...
                    _logevents("$msg: $msghash{0}",0);
                

                } else {
                    @returnarray = _clientcall($ServerName, "build",
                                               "-n", "$buildreq");
                    return["@returnarray"];
                }
                    
            } else {

                #
                # Is this an external command?
                #
                print STDERR "int_ext is E @cmdarray\n";
                if ($int_ext =~ /E/) {
                    my @newarray = _parsearray(@cmdarray);
                    _logevents("$msg\n",0);
                    &$function(@newarray);
                    return[$msghash{0}];
                } else {
                    my @newarray = _parsearray(@cmdarray);
                    # Run subroutine, send back results...
                    my @return = &$function(@newarray);
                    return[@return];
                }    
            }

        } else {

            # Unsupported request...
            return[$msghash{3}];
            if ($msg != "") {
                # Display a message to the console...
                _logevents("\"$msg\": $msghash{3}",1);
            }
        }

        # Undefine msg, wait for new event...
        undef $msg;

        # Display a message to the console...
        $sreqtime = scalar localtime;
        print "[$sreqtime $name]: waiting for events\n";
    } else {
       return [""];
    }

}

#
# Call an external command
#
sub external {

    #
    # Get command and arg list
    #
    my $cmd  = shift;
    my @localarg = @_;

    #
    # Use argv for getopts.
    #
    @ARGV        = @_;

    #
    # Getopts in case we're calling a build
    #
    my %Options;
    my $jobname;
    my $jobno;

    my $ret;

    #
    # Scalar for process id
    #
    my $pid;

    #
    # SQL array use to push values into proctree table
    #
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    #
    # Call getopts
    #
    getopts('n:j:', \%Options);

    #
    # Assign jobname and jobno (if supplied)
    #
    $jobname  = $Options{n};
    $jobno    = $Options{j};

    #
    # Call forkprocess and return with pid
    #
    $pid = $os->forkprocess($cmd, 0, 0, @localarg);

    #
    # If the global build boolean is set then push process values
    # onto the proctree table
    #
    if ($bBuild) {

        #
        # Push values onto array
        #
        push @sqlarray, "$hostname";
        push @sqlarray, "$jobname";

        #
        # If we're sent the jobno then use that value initially
        # otherwise use pid as reference into proctable (it will
        # be replaced by jobno during the build)
        #
        if (defined($jobno)) {
            push @sqlarray, "$jobno";
        } else {
            push @sqlarray, "$pid";
        }

        #
        # Push args sent along with the pid
        #
        push @sqlarray, "$cmd @localarg";
        push @sqlarray, "$pid";

        #
        # Submit values into SQL
        #
        $os->run_sql_submit("proctree", @sqlarray);

        #
        # Set boolean value back to false
        #
        $bBuild = 0;

    }

    return $pid;

}
     
#
# Add user to Perforce server...
#
sub adduser {

    @ARGV = @_;

    my %Options;

    my $port;
    my $user;
    my $first;
    my $last;
    my $group;
    my $mail;

    my $sccs;
    my $command;

    $os->tracefunction((caller(0))[3]);

    getopts('p:u:f:l:g:m:', \%Options);

    $port  = $Options{p};
    $user  = $Options{u};
    $first = $Options{f};
    $last  = $Options{l};
    $group = $Options{g};
    $mail  = $Options{m};

    if (!defined($group)) {
        $group = $config->DEFGROUP;
    }

    if (!defined($mail)) {
        $mail  = "$user\@".$config->COMPANY;
    }

    if (!defined($port) || !defined($user) || !defined($first) ||
        !defined($last)) {
        _logevents("adduser @_: invaild args\n",1);
        return display_command_usage("adduser");
    }

    $sccs = $cmbroker->detectsccs($port);

    $command = $sccs."_useradmin";
    
    my $message = $cmbroker->$command($port, $user, $first, $last, 
                                      $mail, $group, 0);

    if ($message =~ /command failed:/) {
        _logevents("adduser @_: could not add user\n",1);
        return "$message: unable to add user $user to port $port\n";
    } else {
        _logevents("adduser @_: user added\n",0);
        return "$user ($mail) added to $port\n";
    }

    return $message;

}

#
# Remove user from Perforce server...
#
sub deluser {

    @ARGV = @_;

    my %Options;

    my $port;
    my $user;
    my $first;
    my $last;
    my $group;
    my $mail;

    my $sccs;
    my $command;

    $os->tracefunction((caller(0))[3]);

    getopts('p:u:f:l:g:m:', \%Options);

    $port  = $Options{p};
    $user  = $Options{u};
    $first = $Options{f};
    $last  = $Options{l};
    $group = $Options{g};
    $mail  = $Options{m};

    if (!defined($group)) {
        $group = $config->DEFGROUP;
    }

    if (!defined($mail)) {
        $mail  = "$user\@".$config->COMPANY;
    }

    if (!defined($first)) {
        $first = "nobody";
    }

    if (!defined($last)) {
        $last = "nobody";
    }

    $sccs = $cmbroker->detectsccs($port);

    $command = $sccs."_useradmin";

    if (!defined($port) || !defined($user)) {
        _logevents("deluser @_: invaild args\n",1);
        return display_command_usage("adduser");
    }

    my $message = $cmbroker->$command($port, $user, $first, $last, 
                                     $mail, $group, 1);

    if ($message =~ /command failed:/) {
        _logevents("adduser @_: could not remove user\n",1);
        return "$message: unable to delete user $user from port $port\n";
    } else {
        _logevents("deluser @_: user removed\n",0);
        return "$user removed from $port\n";
    }

}

#
# Grab the id of the running process
#
sub getid {

    @ARGV = @_;

    my $ServerName;
    my %Options;

    my @dirarray;
    my @dirarray2;
    my @returnarray;

    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);
   
    getopts('S:', \%Options);

    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        $sqlquery = $qs->formatquery(53, $ServerName, 'server');

        @sqlarray = $os->run_sql_query($sqlquery, ";");

        push @returnarray, "server: $sqlarray[0]\n";

        $sqlquery = $qs->formatquery(53, $ServerName, 'monitor');

        @sqlarray = $os->run_sql_query($sqlquery, ";");

        push @returnarray, "monitor: $sqlarray[0]";

        # Display a message to the console...
        _logevents("getid @_\n",0);
    
    } else {
        @returnarray = _clientcall($ServerName, "getid");
    }

    # Return the command array...
    return @returnarray;

}

#
# Show all buildservers
#
sub servers {

    @ARGV = @_;
    my @options;

    my $ServerName;
    my $bHtml;

    my $entries;
    my $sqlquery;
    my @sqlarray;
    my $state;
    my $line;
    my $string;
    my @return;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:w', \%Options);

    $ServerName = $Options{S};
    $bHtml      = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        $sqlquery = $qs->formatquery(65, $ServerName);
    
        @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
        if ($bHtml) {
        
            foreach $line (@sqlarray) {

                my %tablehash = ();
                my @tabledata = ();

                my ($server, $description, $port, $status) 
                    = split (/;/, $line);

                if ($status) {
                    $state = "up";
                } else {
                    $state = "down";
                }

                %tablehash = (

                    1 => {
                        content => $server,
                        class   => 'values',
                    },
                    2 => {
                        content => $port,
                        class   => 'values',
                    },
                    3 => {
                        content => $state,
                        class   => 'values',
                    },
                    4 => {
                        content => $description,
                        class   => 'values',
                    },

                );

                @tabledata = tableData(%tablehash);
                      
                foreach my $entry (@tabledata) {
                    push @return, $entry;
                }
            }

        } else {
            push @return, "\n";
        
            $string = sprintf ("%-15s %-6s %-6s %-50s\n", "Server Name", 
                               "Port", "State", "Description");
            push @return, $string;
            $string = sprintf ("%-15s %-6s %-6s %-50s\n", "==========", 
                               "=====", "=====", "==========");
            push @return, $string;
        
            foreach $line (@sqlarray) {
        
                my ($server, $description, $port, $status) = split (/;/, $line);
                if ($status) {
                    $state = "up";
                } else {
                    $state = "down";
                }
                my $tmpstring = sprintf("%-15s %-06d %-6s %-50s\n", $server,
                                        $port, $state, $description,);
                push @return, $tmpstring;
        
            }
        }

        # Display a message to the console...
        _logevents("servers\n",0);

    } else {
        if ($bHtml) {
            push @options, "-w";
        }
        @return = _clientcall($ServerName, "servers", @options);
    }

    return @return;

}

#
# Shutdown the build server in a posix environment
#
sub shutdown {

    @ARGV = @_;

    my $entries;
    my $serverid;
    my $monitorid;
    my $sqlquery;
    my @sqlarray;

    my $ServerName;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:', \%Options);

    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        $isinternal = 1;

        my @ids = getid();

        foreach $entries (@ids) {
            my ($left, $right) = split(/:/, $entries);
            chomp $right;
            if ($left =~ /server/) {
                $serverid  = $right;
                $serverid  =~ s/ //g;
            } else {
                $monitorid = $right;
            }
        }
    
        #
        # Have status set to 0 for buildserver
        #
        $sqlquery = $qs->formatquery(89, 0, $hostname);

        @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
        # Display a message to the console...
        _logevents("shutdown\n",0);
    
        if ($POSIX) { 
            $os->forkprocess("kill -s INT $monitorid $serverid", 0, 2);
        } else {
            return $os->shutdown();
        }

    } else {
        _clientcall($ServerName, "shutdown");
    }

    return "shutting down";

}

#
# Print current configuration data loaded from buildconf.pm
#
sub dumpconfig {

    @ARGV = @_;

    my @ret;
    my @options;
    my $line;

    my $ServerName;

    my $bHtml;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:w', \%Options);

    $ServerName = $Options{S};
    $bHtml      = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        if ($bHtml) {
    
            foreach my $key (sort keys %{$config}) {

                my %tablehash = ();
                my @tabledata = ();

                %tablehash = (

                    1 => {
                        content => $key,
                        class   => 'values',
                    },
                    2 => {
                        content => $config->{$key},
                        class   => 'values',
                    },
                );

                @tabledata = tableData(%tablehash);
                      
                foreach my $entry (@tabledata) {
                    push @ret, $entry;
                }
            }

        } else {

            push @ret, "\n";
            push @ret, "See buildconf.pm for comments!\n\n";
    
            foreach my $key (sort keys %{$config}) {
        
                $line = sprintf "%-15s = %s\n", $key, $config->{$key};
                push @ret, $line;
        
            }
        }

        _logevents("dumpconfig", 0);

    } else {
        if ($bHtml) {
            push @options, "-w";
        }
        @ret = _clientcall($ServerName, "dumpconfig", @options);
    }

    return @ret;

}

#
# Generate a process table, internal...
#
sub do_ps {

    @ARGV = @_;

    my $ServerName;
    my $DumpTable;
    my $bHtml;

    my %Options;

    my @psarray;
    my @psargs;

    # Sql declarations
    my $sqlquery;
    my @sqlarray;

    my $entry;

    $os->tracefunction((caller(0))[3]);

    getopts('tS:w', \%Options);

    $ServerName = $Options{S};
    $DumpTable  = $Options{t};
    $bHtml      = $Options{w};

    my $tmpkey;

    my %tmphash = %{ $rpcserver };

    foreach $tmpkey (keys %tmphash) {
        print STDERR "$tmpkey: ".$tmphash{$tmpkey}."\n";
    }

    if ($ServerName =~ /^$hostname$/) {
        undef $ServerName;
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
        # Display a message to the console...
        if ($POSIX) {
            my $pscmd = $config->PS;
            if (!$DumpTable) {
                @psarray = _cmd_array("$pscmd");
                if ($bHtml) {
                    @psarray = _gen_html_ps(@psarray);
                }
            }
        } else {
            if (!$DumpTable) {
                @psarray = $os->processtable();
                if ($bHtml) {
                    @psarray = _gen_html_ps(@psarray);
                }
            }
        }

        if ($DumpTable) {
        
            $sqlquery = $qs->formatquery(54, $ServerName);

            @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

            if (@sqlarray == 0) {
                push @psarray, "no data from process table available";
            } else {

                my $format;

                push @psarray, "SQL Process Table Data for $ServerName:\n\n";
                $format = sprintf("%-12s %-20s %s\n", 'ID', 'Title', 
                                  'Command');
                push @psarray, $format;
                $format = sprintf("%-12s %-20s %s\n", '========', '=========',
                                  '======================');
                push @psarray, $format;

                foreach $entry (@sqlarray) {

                    my ($lpid, $ltitle, $lcmd) = split(/;/, $entry);

                    $format = sprintf("%-12s %-20s %s\n", $lpid, $ltitle, 
                                       $lcmd);
                    push @psarray, $format;

                }

            }
        }

        # Display a message to the console...
        _logevents("ps $ServerName\n",0);
    } else {
        push @psargs, "ps";
        if ($DumpTable) {
            push @psargs, "-t";
        }
        if ($bHtml) {
            push @psargs, "-w";
        }
        @psarray = _clientcall($ServerName, @psargs);
    }

    # Return the command array...
    return @psarray;

}

#
# Kill processes remotely
#
sub do_kill {

    @ARGV = @_;

    my @dirarray;
    my $entry;
    my $exitcode;

    my $ServerName;
    my $Process;
    my $Signal;

    my %Options;

    my @remoteargs = ();

    $os->tracefunction((caller(0))[3]);

    getopts('S:p:s:', \%Options);

    $ServerName = $Options{S};
    $Process    = $Options{p};
    $Signal     = $Options{s};

    if ($ServerName =~ /^$hostname$/) {
        undef $ServerName;
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
        if ($POSIX) {
            @dirarray = _cmd_array("kill -$Signal $Process");
        } else {
            $os->kill($Process);
        }
       _logevents("kill -$Signal $Process\n",0);
    } else {
        push @remoteargs, "kill";
        if (defined($Process)) {
            push @remoteargs, "-p";
            push @remoteargs, "$Process";
        }
        if (defined($Signal)) {
            push @remoteargs, "-s";
            push @remoteargs, "$Signal";
        }
        @dirarray = _clientcall($ServerName, @remoteargs);
    }

    return @dirarray;

}
    
#
# Look at the environment of a running server.
#
sub do_set {

    @ARGV = @_;

    my $ServerName;

    my $bHtml;

    my %Options;

    my @envarray;
    my @argarray;

    my $key;

    $os->tracefunction((caller(0))[3]);

    getopts('S:w', \%Options);

    $ServerName = $Options{S};
    $bHtml      = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        foreach $key (sort keys %ENV) {

            if ($bHtml) {

                my %tablehash = ();
                my @tabledata = ();

                %tablehash = (

                    1 => {
                        content => $key,
                        class   => 'values',
                    },
                    2 => {
                        content => $ENV{$key},
                        class   => 'values',
                    },
                );

                @tabledata = tableData(%tablehash);
                      
                foreach my $entry (@tabledata) {
                    push @envarray, $entry;
                }
            } else {
                push @envarray, "$key = $ENV{$key}\n";
            }

        }
        _logevents("set @_\n",0);

    } else {
        if ($bHtml) {
            push @argarray, "-w";
        }
        @envarray = _clientcall($ServerName, "set", @argarray);
    }

    return @envarray;

}

#
# General function for reading a file...
#
sub _read_file_log {

    # Make this more readable...
    my $BuildName = shift;
    my $BuildType = shift;
    my $LogType   = shift;

    my $LogFile;
    my $line;

    my @dirarray;

    #
    # Call subroutine to look up the build directory...
    #
    my $BuildRoot = _getbuildroot($BuildName);

    $os->tracefunction((caller(0))[3]);

    # print "BuildRoot = $BuildRoot\n";

    #
    # If we didn't find a buildroot...
    #
    if (!defined($BuildRoot)) {
        return "no build $BuildName found\n";
    }

    #
    # If there is a file we need to pass it's contents back
    # to the client...
    #
    if (($LogType =~ /error/) || ($LogType =~ /remote/)) {
        if ($LogType =~ /error/) {
            $LogFile = "$BuildName.$BuildType.$LogType.log";
        } else {
            $LogFile = "$BuildName.$BuildType.sync.log";
        }
    } else {
        $LogFile = "$BuildName.$LogType.log";
    }

    if (-f "$BuildRoot/$LogFile") {

        # Get contents of the errorlog...
        @dirarray = _catfile("$BuildRoot/$LogFile");

        foreach $line (@dirarray) {
            if ($line =~ /File\(s\) up-to-date/) {
                return " ";
            }
        }

        unshift(@dirarray,"\n");

        # Return the array...
        return @dirarray;
    } else {
        # There is no error log...
        return "no $LogType.log for $BuildName $BuildType found!\n";
    }

}

# Subroutine to return updates to the build server
sub whatsnew {

    @ARGV = @_;

    my $ServerName;

    my @FileArray;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:', \%Options);

    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {
        # Get contents of the build.txt file...
        if ( -f "$dir/updates/$hostname.whatsnew.txt") {
            @FileArray = _catfile("$dir/updates/$hostname.whatsnew.txt");
            _logevents("whatsnew $ServerName\n",0);
            return @FileArray;
        } else {
            _logevents("whatsnew $ServerName.whatsnew.txt\n",1);
            return "no update file found for $hostname\n";
        }
    } else {
        @FileArray = _clientcall($ServerName, "whatsnew");
    }

    return @FileArray;

}

#
# Subroutine to return instructions on running a build locally (if any)
#
sub instructions {

    @ARGV = @_;
    
    my $ServerName;
    my $BuildName;
    my @FileArray;
  
    my @argarray;

    my %Options;

    $os->tracefunction((caller(0))[3]);
    
    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    # Make sure we have a buildname
    if (!defined($BuildName)) {
        _logevents("instructions @_: missing buildname\n",1);
        return display_command_usage("instructions");
    }

    if ($ServerName =~ /^$hostname$/) {
        #
        # Get contents of the build.txt file...
        #
        if ( -f "$dir/build_docs/$BuildName.txt") {
            @FileArray = _catfile("$dir/build_docs/$BuildName.txt");
            _logevents("instructions $BuildName $ServerName\n",0);
        } else {
            _logevents("instructions $BuildName $ServerName\n",1);
            push @FileArray, "no instructions for $BuildName found!"
        }
    } else {
        push @argarray, "instructions";
        push @argarray, "-n";
        push @argarray, "$BuildName";

        @FileArray = _clientcall($ServerName, @argarray);
    } 

    return @FileArray;

}

# Subroutine to return the contents of the synclog...
sub getsync_log {

    @ARGV = @_;

    my $ServerName;

    # Make this more readable...
    my $BuildName;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};

    my $BuildRoot = _getbuildroot($BuildName);

    my $rserver;
    my $rclient;
    my $file;
    my $ext;

    my @FileArray;
    my @RFileArray;

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {
        #
        # Make sure we have a buildname
        #
        if (!defined($BuildName)) {
            _logevents("synclog @_: missing buildname\n",1);
            return display_command_usage("synclog");
        }
    
        @FileArray = _read_file_log($BuildName, "sync", "sync");
    
        _logevents("synclog $BuildName.sync.log\n",0);
    } else {
        @FileArray = _clientcall($ServerName, "synclog", "-n", "$BuildName");
    }

    return "@FileArray @RFileArray";

}

# Subroutine to retturn the contents of the errorlog (if any)
sub geterror_log {

    @ARGV = @_;

    # Make this more readable...
    my $BuildName;
    my $BuildType;
    my $ServerName;

    my %Options;

    my @argarray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:t:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $BuildType  = $Options{t};

    my @FileArray;

    # Make sure we have a buildname
    if (!defined($BuildName)) {
        _logevents("errorlog @_: missing buildname\n",1);
        return display_command_usage("errorlog");
    }

    # Make sure we have a buildtype
    if (!defined($BuildType)) {
        _logevents("errorlog @_: missing buildtype\n",1);
        return display_command_usage("errorlog");
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {
        @FileArray = _read_file_log($BuildName, $BuildType, "error");
    } else {
        push @argarray, "errorlog";
        push @argarray, "-n";
        push @argarray, "$BuildName";
        push @argarray, "-t";
        push @argarray, "$BuildType";
        push @argarray, "error";

        @FileArray = _clientcall($ServerName, @argarray);
    } 

    _logevents("errorlog $ServerName $BuildName $BuildType\n",0);

    return @FileArray;

}

sub do_joblog {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my @JobInfo;

    my %Options;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $line;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};

    if (!defined($BuildName)) {
        _logevents("joblog: missing buildname\n",1);
        return display_command_usage("joblog");
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("joblog: $BuildName $ServerName $msghash{2}",1);
        return $msghash{2};
    }

    if ($ServerName =~ /^$hostname$/) {

        _logevents("joblog: $BuildName $ServerName\n",0);
    
        $sqlquery = (47, $ServerName, $BuildName);
    
        @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
    
        if (@sqlarray == 0) {
            return "No joblog for $BuildName on $ServerName\n";
        } 
    
        foreach $line (@sqlarray) {
            push @JobInfo, "$line\n";
        }
    
        unshift(@JobInfo,"\n");

    } else {
        @JobInfo = _clientcall($ServerName, "-n", "$BuildName");
    }

    return @JobInfo;
}

sub debuglog {

    @ARGV = @_;

    my $BuildName;
    my $ServerName;

    my %Options;

    my @FileArray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $BuildName  = $Options{n};
    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {
        @FileArray = _catfile("$Mtmpdir/$BuildName.log");
        unshift(@FileArray, "\n");
    } else {
        @FileArray = _clientcall($ServerName, "debuglog", "-n", "$BuildName");
    }

    _logevents("debuglog $ServerName $BuildName\n",0);

    return @FileArray;

}

sub getbuild_log {

    @ARGV = @_;

    my $BuildName;
    my $BuildType;
    my $ServerName;

    my $bHtml = 0;

    my %Options;

    my @FileArray     = ();
    my @argarray      = ();

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:t:w', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $BuildType  = $Options{t};
    $bHtml      = $Options{w};
    
    if (!defined($BuildName)) {
        _logevents("buildlog @_: missing buildname\n",1);
        return display_command_usage("buildlog");
    }

    if (!defined($BuildType)) {
        _logevents("buildlog @_: missing buildtype\n",1);
        return display_command_usage("buildlog");
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        @FileArray = _read_file_log($BuildName, $BuildType, $BuildType);

        if ($bHtml) {
            @FileArray = _genHtmlBuildLog($BuildName, @FileArray);
        } 

    } else {

        push @argarray, "buildlog";
        push @argarray, "-n";
        push @argarray, "$BuildName";
        push @argarray, "-t";
        push @argarray, "$BuildType";

        if ($bHtml) {
            push @argarray, "-w";
        }

        @FileArray = _clientcall($ServerName, @argarray);
    } 

    _logevents("buildlog $BuildName $BuildType $ServerName\n",0);

    return @FileArray;

}

sub jobstate {
    
    @ARGV = @_;

    my %Options;

    my $c=0;
    my @locks;
    my @jobArray;
    my @JOBArray;

    my $ServerName;

    $JOBArray[$c] = "\n";
    $c++;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $line;

    # stuff found by strict
    my $lock;

    $os->tracefunction((caller(0))[3]);

    getopts('S:', \%Options);

    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (_serverup($ServerName)) {

        # Grab a list of jobs from the buildtable...
        @jobArray = _getjoblist($ServerName);
    
        # For each entry returned, check to see if a lock exists.
        foreach $lock (@jobArray) {
    
            $sqlquery = $qs->formatquery(64, 1, $lock, 0, $ServerName);
    
            @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
            if (@sqlarray > 0) {
                foreach $line (@sqlarray) {
                    $JOBArray[$c] = $line;
                    $locks[$c]    = "$JOBArray[$c]\n";
                }
    
                $sqlquery = $qs->formatquery(50, $ServerName, $lock, 2);
    
                @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
    
                foreach $line (@sqlarray) {
                    chomp $JOBArray[$c];
                    $locks[$c] = "$JOBArray[$c] - status change pending!\n";
                }
    
                $c++;
    
            }
    
    
        }
        
        if (defined($locks[$c-1])) {
            chomp $locks[$c-1];
        }
        _logevents("jobstate\n",0);
        if ($c == 1) {
            return "buildserver $ServerName is currently idle\n";
        } else {
            return @locks;
        }
    
    } else {
        _logevents("jobstate $ServerName\n",0);
        return "buildserver $ServerName is currently down\n";
    }

}
    
sub clone {

    @ARGV = @_;

    my $oldjob;
    my $newjob;
    my $ServerName;

    my %Options;

    my $ret;

    my $sqlquery;
    my @sqlarray;
    my @sqlsubmit;

    my $newpath;
    my $oldpath;

    my $changestring;

    my $key;
    my $Is;
    my $Os;

    my $jobdir = $config->JOBDIR;

    my %scripts = (
        "retail",        $config->RETAIL,
        "debug",         $config->DEBUG,
        "onfail",        $config->ONFAIL,
        "prebuild",      $config->PREBUILD,
        "postbuild",     $config->POSTBUILD,
        "postpostbuild", $config->POSTPOSTBUILD
    );

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:c:', \%Options);
    
    $ServerName = $Options{S};
    $oldjob     = $Options{n};
    $newjob     = $Options{c};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($oldjob)) {
        _logevents("clone: $msghash{9}",1);
        return display_command_usage("clone");
    }

    if (!defined($newjob)) {
        _logevents("clone: $msghash{9}",1);
        return display_command_usage("clone");
    }

    if (!_ValidJob($oldjob,$ServerName,0)) {
        _logevents("clone: $oldjob $msghash{2}",1);
        return "Unknown job $oldjob\n";
    }

    if (_ValidJob($newjob,$hostname,0)) {
        _logevents("clone: $newjob $msghash{17}",1);
        return "$newjob already exists\n";
    }

    $sqlquery = $qs->formatquery(30, $oldjob, $ServerName);

    @sqlarray = $os->run_sql_query("$sqlquery", ",", 0);

    $changestring = $sqlarray[0];

    $changestring =~ s/$oldjob/$newjob/g;
    $changestring =~ s/$ServerName/$hostname/g;

    @sqlsubmit = split(/,/,$changestring);

    if (( $ret = $os->run_sql_submit("configuration", @sqlsubmit)) == 0) {
        _logevents("clone $oldjob $newjob: SQL failed\n",1);
        return "Failed to write configuration info to SQL server\n";
    }

    #
    # Create newpath / oldpath for generation of build scripts
    #
    $newpath  = $config->BSR;
    $newpath .= "/jobs/$newjob";

    $oldpath  = $config->BSR;
    $oldpath .= "/jobs/$oldjob";
    
    if (! -d "$newpath") {
        mkpath("$newpath", 0, 0755);
    }

    if (-d "$oldpath") {
        foreach $key (keys %scripts) {

            $Is = $oldpath."/".$scripts{$key};
            $Os = $newpath."/".$scripts{$key};

            _copycontents($Is, $Os, $oldjob, $newjob);

        }
    }

    #
    # Make output directory for new job
    #
    if (! -d "$jobdir/$newjob") {
        mkpath("$jobdir/$newjob", 0, 0755);
    }

    $Is  = $config->BSR;
    $Is .= "/build_docs/$oldjob.txt";
    $Os  = $config->BSR;
    $Os .= "/build_docs/$newjob.txt";

    _copycontents($Is, $Os, $oldjob, $newjob);

    $Is  = $config->BSR;
    $Is .= "/config/$oldjob.errors";
    $Os  = $config->BSR;
    $Os .= "/config/$newjob.errors";

    _copycontents($Is, $Os, $oldjob, $newjob);

    _logevents("clone $oldjob $ServerName $newjob $hostname\n",0);

    return "job $newjob on $hostname cloned from $oldjob on $ServerName\n";

}

#
# Allow user permissions to checkin to tree for a particular job
#
sub treeperms { 

    #
    # Declarations for args..
    #
    @ARGV = @_;

    my %Options;

    my $jobname;
    my $modulename;
    my $userid;
    my $bAdd;
    my $bRemove;
    my $bList;

    my $sccs;
    my $port;
    my $command;
    my $arl;

    my $ServerName;

    my $recs;

    my @return;

    #
    # Declarations for sql queries....
    #
    my $sqlquery;
    my @sqlarray;
    my @argarray;

    $os->tracefunction((caller(0))[3]);

    #
    # Read options
    #
    getopts('S:n:m:u:arl', \%Options);
    
    $ServerName = $Options{S};
    $jobname    = $Options{n};
    $modulename = $Options{m};
    $userid     = $Options{u};
    $bAdd       = $Options{a};
    $bRemove    = $Options{r};
    $bList      = $Options{l};

    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    } else {
        push @argarray, "treeperms";
    }

    #
    # Test for modulename...
    #
    if (!defined($modulename)) {
        _logevents("treeperms: $msghash{9}",1);
        return display_command_usage("treeperms");
    }

    push @argarray, "-m";
    push @argarray, "$modulename";

    #
    # Test for combinations of args...
    #
    if (defined($bAdd) && defined($bRemove)) {
        _logevents("treeperms: $msghash{9}",1);
        return display_command_usage("treeperms");
    }

    #
    # Test for combinations of args...
    #
    if (defined($bAdd) && defined($bList)) {
        _logevents("treeperms: $msghash{9}",1);
        return display_command_usage("treeperms");
    }

    #
    # Test for combinations of args...
    #
    if (defined($bRemove) && defined($bList)) {
        _logevents("treeperms: $msghash{9}",1);
        return display_command_usage("treeperms");
    }

    #
    # Test for combinations of args...
    #
    if (!defined($bRemove) && !defined($bList) && !defined($bAdd)) {
        _logevents("treeperms: $msghash{9}",1);
        return display_command_usage("treeperms");
    }
    
    #
    # Make sure it's a valid active job
    #
    if (!_ValidJob($jobname,$ServerName,0)) {
        _logevents("treeperms $jobname $msghash{2}",1);
        return ("undefined job $jobname");
    }
    
    if ($bAdd) {
        $arl = 1;
        push @argarray, "-a";
    }

    if ($bRemove) {
        $arl = 2;
        push @argarray, "-r";
    }
    
    if ($bList) {
        $arl = 3;
        push @argarray, "-l";
    }

    if (defined($userid)) {
        push @argarray, "-u";
        push @argarray, "$userid";
    }

    if (defined($jobname)) {
        push @argarray, "-n";
        push @argarray, "$jobname";
    }

    if ($ServerName =~ /^$hostname$/) {

        $sqlquery = $qs->formatquery(63, $jobname, $hostname);
    
        @sqlarray = $os->run_sql_query($sqlquery, ",", 0);
    
        $recs = @sqlarray;
    
        if (!$recs) {
            return "error reading job entry from sql!!!!\n";
        } else {
            ($sccs, $port) = split(/,/, $sqlarray[0]);
        }
    
        #
        # Use abstraction for stdoutupdate
        #
        $command = $sccs."_treeperms";
    
        @return  = $cmbroker->$command($jobname, $modulename, $port, 
                                       $arl, $userid);
    
        # Display a message to the console...
        _logevents("treeperms @_\n",0);

    } else {

        @return = _clientcall($ServerName, @argarray);
    }
 
    return @return;
    
}

sub lock_build {

    @ARGV = @_;

    my $lock;
    my $myret;
    my $ServerName;

    my %Options;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;
    my @sqlsubmit;
    my @jobarray;
    my $line;
    my $ret;

    my @return;

    my $itsnow = $os->_get_hosttime();

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $lock       = $Options{n};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($lock)) {
        _logevents("disable: $msghash{9}",1);
        return display_command_usage("disable");
    }

    if ($ServerName =~ /^$hostname$/) {

        if ($lock !~ /^all$/ic) {
            if (!_ValidJob($lock,$ServerName,0)) {
                _logevents("disable: $lock $msghash{2}",1);
                return "Unknown job $lock on $ServerName\n";
            }
        }
    
        if ($lock =~ /^all$/ic) {
                @jobarray = _getactivejobs($ServerName);
        } else {
            push @jobarray, $lock;
        }
    
        foreach $line (@jobarray) {
    
            @sqlsubmit = ();
            @sqlarray  = ();
    
            $sqlquery = $qs->formatquery(69, $ServerName, $line, 1);
    
            @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
            if (@sqlarray == 0) {
    
                push @sqlsubmit, "$ServerName";
                push @sqlsubmit, "$line";
                push @sqlsubmit, "=[$line build job disabled]=";
                push @sqlsubmit, "$itsnow";
                push @sqlsubmit, "1";
    
                if (($ret = $os->run_sql_submit("semaphores", 
                    @sqlsubmit)) == 0) {
                    $myret .= "SQL failed";
                    _logevents("disable: SQL failed for $line\n",1);
                } else {
                    $myret .= "job $line disabled";
                    _logevents("disable: $line\n",0);
                }
    
            } else {
    
                $sqlquery = $qs->formatquery(69, $ServerName, $line, 2);
    
                @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
                if (@sqlarray == 0) {
    
                    push @sqlsubmit, "$ServerName";
                    push @sqlsubmit, "$line";
                    push @sqlsubmit, "=[$line build job disabled]=";
                    push @sqlsubmit, "$itsnow";
                    push @sqlsubmit, "2";
        
                    if (($ret = $os->run_sql_submit("semaphores", 
                         @sqlsubmit)) == 0) {
                        $myret .= "SQL pending failed";
                        _logevents("disable: $line SQL pending failed\n",1);
                    } else {
                        $myret .= "$line lock pending";
                        _logevents("$line: $msghash{15}",0);
                    }
                }
    
            }
    
        } 
        push @return, $msghash{0};
    } else {
        @return = _clientcall($ServerName, "disable", "-n", "$lock");
    }

    return @return;

}

sub _getactivejobs {

    my $ServerName = shift;

    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(80, $ServerName, 0);

    @sqlarray = $os->run_sql_query($sqlquery,",");

    return @sqlarray;

}

sub unlock_build {

    @ARGV = @_;

    my $job;
    my $myret;

    my @return;

    my $ServerName;
    my $force;
    my $kill;

    my @jobarray;

    my @argarray;
    my @psarray;
    my $process;

    my %Options;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;
    my $msgquery;
    my @msgarray;
    my $buildnum;
    my $jobdir = $config->JOBDIR;;
    my $line;
    my $ret;

    #
    # Get rid of any earlier data stored away here.
    #
    %processtree = ();
    %foundhash   = ();

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:fk', \%Options);

    $ServerName = $Options{S};
    $job        = $Options{n};
    $force      = $Options{f};
    $kill       = $Options{k};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($job)) {
        _logevents("enable\n",1);
        return display_command_usage("enable");
    }

    if ($ServerName =~ /^$hostname/) {

        if ($job =~ /^all$/ic) {
            @jobarray = _getactivejobs($ServerName);
        } else {
            push @jobarray, $job;
        }
    
        foreach $line (@jobarray) {

            $msgquery = $qs->formatquery(49, $ServerName, $line, "in progress");
            @msgarray = $os->run_sql_query("$msgquery", ";", 0);
            $buildnum = _get_build_num($msgarray[0]);

            #
            # This shouldn't mater if 'all' is specified, but for a 
            # single job, it will return an invalid job message...
            #
            if (!_ValidJob($line,$ServerName,0)) {
                _logevents("enable: $line $msghash{2}",1);
                return "Unknown job $line on $ServerName\n";
            }
    
            @sqlarray  = ();
    
            $sqlquery = $qs->formatquery(69, $ServerName, $line, 1);
    
            @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
            if (@sqlarray == 0) {
                $myret .= "$reqtime $peeraddress: enable $job $msghash{0}";
                _logevents("$myret",0);
            } else {
                $sqlquery = $qs->formatquery(4, $ServerName, $line);
    
                $ret = $os->run_sql_remove("$sqlquery");
    
                if (!$ret) {
                    $myret .= "$reqtime $peeraddress SQL failed: enable ".
                              "$line\n";
                    _logevents("$myret",1);
                } else {
                    $myret .= "$reqtime $peeraddress: enable ".
                              "$line $msghash{2}";
                    _logevents("$myret",1);
                }
                if (!$force) {
                    $isinternal = 1;
                    recover("-n", "$line");
                    $isinternal = 0
                } else {
                    $sqlquery = $qs->formatquery(4, $ServerName, $line);
                    $ret = $os->run_sql_remove("$sqlquery");
                }
                if ($kill && $force) {
                    my $id;
                    my @cleanargs;
                    my $rm;
                    my $pid;

                    my $command;
                    my @cmdarray;

                    my $cmquery;
                    my @cmarray;

                    my $rollback;

                    $sqlquery = $qs->formatquery(52, $line, $ServerName);
                    @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

                    #
                    # Need to clean up any records left in the
                    # database.
                    #
                    push @cleanargs, "-n";
                    push @cleanargs, "$line";
                    push @cleanargs, "-j";
                    push @cleanargs, "$buildnum";
                    push @cleanargs, "-f";

                    #
                    # Call the function to remove job info
                    #
                    $isinternal = 1;
                    cleanjob(@cleanargs);
                    $isinternal = 0;
                            
                    if (@sqlarray == 0) {
                        _logevents("$reqtime $peeraddress no pid to kill",1);
                    } else {

                        #
                        # Remove info from the restore table
                        #
                        $sqlquery = $qs->formatquery(115, $ServerName, $line);
                        $ret = $os->run_sql_remove("$sqlquery");

                        foreach $id (@sqlarray) {

                            if ($POSIX) {
                                $os->killall($id);
                            }

                        }

                        #
                        # Remove all the process entries from protree
                        # since they've all (supposedly) been killed.
                        #
                        $sqlquery = $qs->formatquery(3, $line, $ServerName);
                        $os->run_sql_remove("$sqlquery");

                    }

                    $cmquery = $qs->formatquery(55, $ServerName, $line);
                    @cmarray = $os->run_sql_query("$cmquery", ";", 0);

                    my ($cmp, $cmc, $cmt, $cms) = split(/;/, $cmarray[0]);

                    $rollback = _get_last_job($line);

                    if(defined($rollback)) {
                        $command  = $cms."_rollback";
                        @cmdarray = $cmbroker->$command($cmc, $cmp, $cmt,
                                                        $rollback, $dir);
                    }

                    #
                    # Remove the build directory if it exists
                    #
                    if ( -d "$jobdir/$line/$buildnum") {
                        if ($POSIX) {
                            $rm = "rm -rf";
                        } else {
                            $rm = "rd /s/q/y";
                        }
                        $pid = $os->forkprocess(
                               "$rm $jobdir/$line/$buildnum", 0, 0);
                    }

                }

            }
    
        } 
        push @return, $msghash{0};

    } else {
        push @argarray, "enable";
        push @argarray, "-n";
        push @argarray, "$job";
        if ($force) {
            push @argarray, "-f";
        }
        if ($kill && $force) {
            push @argarray, "-k";
        }

        @return = _clientcall($ServerName, @argarray);
    }

    return @return;

}

sub get_fails {

    @ARGV = @_;

    my @BLDArray;
    my @TMPArray;
    my @argarray;
    my $result;
    my $elements;
    my $ret;

    my $ServerName;
    my $jobName;
    my $showlast;

    my %Options;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $line;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:l:', \%Options);

    $ServerName = $Options{S};
    $jobName    = $Options{n};
    $showlast   = $Options{l};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($jobName)) {
        _logevents("incomplete: $msghash{2}",1);
        return display_command_usage("incomplete");
    }

    if (!_ValidJob($jobName,$ServerName,0)) {
        _logevents("incomplete $jobName: $msghash{2}",1);
        return "Unknown job $jobName\n";
    }

    if ($ServerName =~ /^$hostname/) {
        $sqlquery = $qs->formatquery(38, $ServerName, $jobName, 2);
    
        @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
    
        if (@sqlarray > 0) {
            push @BLDArray, "\n";
        }
        foreach $line (@sqlarray) {
            chomp $line;
            push @BLDArray, "$line\n";
        }
    
        my $elements = @BLDArray;
    
        if ($elements > 0) {
    
            if (defined($showlast)) {
                if ($elements > $showlast) {
                    $result = $elements - $showlast;
                    @TMPArray = splice(@BLDArray, $result, $showlast);
                    unshift(@TMPArray, "\n");
                    return @TMPArray;
                } else {
                    return @BLDArray;
                }
            }
    
        } else {
            push @BLDArray, "no build failures for $jobName on $ServerName";
        }

        _logevents("incomplete $jobName $ServerName\n",0);
    } else {
        push @argarray, "-n";
        push @argarray, "$jobName";
        if (defined($showlast)) {
            push @argarray, "-l";
            push @argarray, "$showlast";
        }
        @BLDArray = _clientcall("$ServerName", "incomplete", @argarray);
    }

    return @BLDArray;

}

sub xmldump {

    @ARGV = @_;

    my $jobname;
    my $jobno = 0;
    my $server;
    my $limit = 0;
    my $bFiles = 0;
    my $bComments = 0;
    my $bTeststat = 0;
    my $bAllservers = 0;
    my $bAlljobs = 0;
    my $bRunning = 0;
    my $formatted;
    my $ServerName;
    my $bPromoted;

    my @return;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('n:s:l:j:S:prfct', \%Options);

    $jobname    = $Options{n};
    $jobno      = $Options{j};
    $server     = $Options{s};
    $limit      = $Options{l};
    $ServerName = $Options{S};
    $bFiles     = $Options{f};
    $bComments  = $Options{c};
    $bTeststat  = $Options{t};
    $bRunning   = $Options{r};
    $bPromoted  = $Options{p};

    #
    # For use with mysql
    #
    my $sqlquery;
    my @sqlarray;

    if ($jobname =~ /^all$/) { 
        $bAlljobs = 1; 
    }

    if ($server =~ /^all$/) { 
        $bAllservers = 1; 
    }

    if ($jobno) { 
        $limit = 0; 
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($server)) {
        $server = $hostname;
    }

    if (!defined($jobname)) {
        _logevents("xmldump: $msghash{2}",1);
        return "missing job name\n"; 
    }

    if (!$bAllservers && !$bAlljobs) {
        my $ret;
        if (($ret = _ValidJob($jobname,$server,0)) == 0) {
            _logevents("xmldump: $jobname $msghash{2}",1);
            return "undefined job $jobname on $server\n";
        }
    }

    my $entries;

    if ($ServerName =~ /^$hostname$/) {

        if ($bAlljobs) {
            $sqlquery = $qs->formatquery(85, 0);
        } else {
            $sqlquery = $qs->formatquery(86, 0, $jobname);
        }
    
        if (!$bAllservers) {
            $sqlquery .= "and server=\"$server\" ";
        }
    
        @sqlarray = $os->run_sql_query($sqlquery, ";");
    
        $entries = @sqlarray;
    
        if (!$entries) {
            return "No job $jobname found\n";
        }

        @return = _xmljobinfo($bFiles, $bComments, $bTeststat, $bRunning, 
                              $bPromoted, $limit, $jobno, @sqlarray);

        $entries = @return;

        if (!$entries) {
            return "No records returned for $jobname\n";
        }
    
        _logevents("xmldump: $jobname $server\n",0);

    } else {

        my   @argarray;

        push @argarray, "xmldump";
        push @argarray, "-n";
        push @argarray, "$jobname";

        if ($server !~ /$hostname/ic) {
            push @argarray, "-s";
            push @argarray, "$server";
        }
        
        if ($jobno) {
            push @argarray, "-j";
            push @argarray, "$jobno";
        }

        if ($bFiles) {
            push @argarray, "-f";
        }

        if ($bComments) {
            push @argarray, "-c";
        }

        if ($bTeststat) {
            push @argarray, "-t";
        }

        if ($bRunning) {
            push @argarray, "-r";
        }

        if ($bPromoted) {
            push @argarray, "-p";
        }

        @return = _clientcall($ServerName, @argarray);

    }

    return @return;

}

sub testserver {

    @ARGV = @_;

    my %Options;
    my @argarray;
    my @return;

    my $sqlquery;
    my @sqlarray;

    my $list;
    my $server;
    my $ServerName;

    my $line;
    my $infoline;

    $os->tracefunction((caller(0))[3]);

    getopts('lf:S:', \%Options);

    $list       = $Options{l};
    $server     = $Options{f};
    $ServerName = $Options{S};

    if ($list && defined($server)) {
        _logevents("testserver: $msghash{7}",1);
        return display_command_usage("authorize");
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        my $recs;

        if ($list) {

            $sqlquery = $qs->formatquery(36);
            @sqlarray = $os->run_sql_query($sqlquery, ";");
            $recs = @sqlarray;

            if (!$recs) {
                _logevents("testserver: no records found", 0);
                return "no records found\n";
            }

            $infoline = sprintf("%-30s %-10s\n", "Server", "State");
            push @return, $infoline;

            $infoline = sprintf("%-30s %-10s\n", "===============", 
                                "========");
            push @return, $infoline;
            
            foreach $line (@sqlarray) {
                my ($name, $state) = split(/;/, $line);
                if ($state) {
                    $infoline = sprintf("%-30s %-10s\n", $name, "happy");
                    push @return, $infoline;
                } else {
                    $infoline = sprintf("%-30s %-10s\n", $name, "unhappy");
                    push @return, $infoline;
                }
            }

            _logevents("testserver: processed $recs records", 0);

        }

        if (defined($server)) {

            $sqlquery = $qs->formatquery(67, 0, $server);
            @sqlarray = $os->run_sql_query($sqlquery, ";");

            $recs = @sqlarray;

            if (!$recs) {
                _logevents("testserver: no records to free found", 0);
                return "no server to free up found\n";
            }

            $sqlquery = $qs->formatquery(91, 1, $server);
            @sqlarray = $os->run_sql_query($sqlquery, ";");

            push @return, "$server free'd for testing\n";

            _logevents("testserver: $server freed", 0);
 
        }
   

    } else {

        push @return, "testserver";

        if ($list) {
            push @return, "-l";
        }

        if (defined($server)) {
            push @return, "-f";
            push @return, "$server";
        }

        @return = _clientcall($ServerName, @argarray);

    }

    return @return;

}

sub teststatus {

    @ARGV = @_;

    my $jobname;
    my $jobno;
    my $limit;
    my $pending;
    my $completed;
    my $running;
    my $bHtml;

    my $ServerName;

    my $displaytype;
    my $kind;
    my $entries;

    my $infoline;
    my $line;
    my $limitto = 'order by job desc';

    my $bAlljobs = 0;

    my @return;

    my %Options;

    #
    # Hash to store tables required for request.
    #
    my %tablehash;
    my %recordhash;

    #
    # SQL declarations
    #
    my $primaryquery;
    my @primaryarray;
    my $sqlquery;
    my @sqlarray;

    my @argarray;

    #
    # FIXME Need to generalize this or add it to the configuration
    #
    my %whathash = (
        cp   => 'Platform',
        mail => 'Mailserver'
    );

    $os->tracefunction((caller(0))[3]);

    getopts('n:j:l:S:pcrw', \%Options);

    $jobname    = $Options{n};
    $jobno      = $Options{j};
    $limit      = $Options{l};
    $ServerName = $Options{S};
    $pending    = $Options{p};
    $completed  = $Options{c};
    $running    = $Options{r};
    $bHtml      = $Options{w};

    if (!defined($jobname)) {
        _logevents("teststatus: $msghash{2}",1);
        return "missing job name\n"; 
    }

    push @argarray, "teststatus";
    push @argarray, "-n";
    push @argarray, "$jobname";

    #
    # Results table is the default
    #
    if (!$pending && !$completed && !$running) {
        $tablehash{results} = "Completed"
    }

    #
    # Put appropriate tble into hash
    #
    if (defined($jobno)) {
        $tablehash{run}     = "Running";
        $tablehash{stage}   = "Pending";
        $tablehash{results} = "Completed";
    } else {
        if ($running) {
            push @argarray, "-r";
            $tablehash{run}     = "Running";
        }
        
        if ($pending) {
            push @argarray, "-p";
            $tablehash{stage}   = "Pending";
        }
        
        if ($completed) {
            push @argarray, "-c";
            $tablehash{results} = "Completed";
        }

    }

    if ($jobname =~ /^all$/) { 
        $bAlljobs = 1; 
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!$bAlljobs) {
        my $ret;
        if (($ret = _ValidJob($jobname,$ServerName,0)) == 0) {
            _logevents("teststatus: $jobname $msghash{2}",1);
            return "undefined job $jobname on $ServerName\n";
        }
    }

    if ($limit) {
        push @argarray, "-l";
        push @argarray, "$limit";
        $limitto .= " limit $limit";
    }

    if (defined($jobno) && $bAlljobs) {
        _logevents("teststatus: $jobname $jobno $msghash{7}",1);
        return "cannot mix all jobs with job number";
    }

    if ($ServerName =~ /^$hostname$/) {

        #
        # Push headers onto return array;
        #
        if (!$bHtml) {
            $infoline = sprintf("%-14s %-12s %-18s %-8s %-10s %-12s\n", 
                                "Buildname", "Jobno", "Testname", "OS", 
                                "Server", "State");
            push @return, $infoline;

            $infoline = sprintf("%-14s %-12s %-18s %-8s %-10s %-12s\n",
                                "=============", "===========", 
                                "==================", "========", 
                                "==========", "==========");
            push @return, $infoline;

        }

        if (!$bAlljobs) {
            $primaryquery = $qs->formatquery(79, $ServerName, $jobname);
        } else {
            $primaryquery = $qs->formatquery(78, $ServerName);
        }

        @primaryarray = $os->run_sql_query($primaryquery, ":");
    
        $entries = @primaryarray;
    
        if (!$entries) {
            if (!$bHtml) {
                return "No records found\n";
            } else {
                return "";
            }
        }

        foreach $line (@primaryarray) {

            my ($title, $comment, $target, $plat, $type, $buildserver,
                $buildname, $dependserver) = split(/:/, $line);


            if ($title =~ /mail/) {
                $kind = $whathash{mail};
            } else {
                $kind = $whathash{cp};
            }

            foreach my $key (keys %tablehash) {

                if (defined($jobno)) {
                    $sqlquery = $qs->formatquery(41, $key, $buildserver, 
                                                 $buildname, $target, $title, 
                                                 "and job='$jobno'");
                } else {
                    $sqlquery = $qs->formatquery(41, $key, $buildserver, 
                                                 $buildname, $target, $title, 
                                                 $limitto);
                }

                @sqlarray = $os->run_sql_query($sqlquery, ";");

                if (@sqlarray) {

                    foreach my $entry (@sqlarray) {

                        %{ $recordhash{"$entry:$type:$kind:$target:$plat"} }= (
                            NAME     => "$buildname",
                            ENTRY    =>  $entry,
                            TESTNAME =>  $title,
                            PLATFORM =>  $plat,
                            TARGET   =>  $target,
                            TABLE    =>  $tablehash{$key},
                        );
                    }

                }
            }

        }

        #
        # Properly sort records...
        #
        if ($bHtml) {

            foreach my $record (reverse sort keys %recordhash) {

                my %tablehash = ();
                my @tabledata = ();

                %tablehash = (

                    1 => {
                        content => $recordhash{$record}{NAME},
                        class   => 'values',
                    },
                    2 => {
                        content => $recordhash{$record}{ENTRY},
                        class   => 'values',
                    },
                    3 => {
                        content => "<a href=/editor.php?server=$hostname&name=".
                                   $recordhash{$record}{NAME}."&testname=".
                                   $recordhash{$record}{TESTNAME}."&action=".
                                   "edit&table=testconfiguration&user=1".
                                   "&access=login>".
                                   $recordhash{$record}{TESTNAME}."</a>",
                        class   => 'values',
                    },
                    4 => {
                        content => $recordhash{$record}{PLATFORM},
                        class   => 'values',
                    },
                    5 => {
                        content => $recordhash{$record}{TARGET},
                        class   => 'values',
                    },
                    6 => {
                        content => $recordhash{$record}{TABLE},
                        class   => 'values',
                    },
                    
                );

                @tabledata = tableData(%tablehash);
                      
                foreach my $entry (@tabledata) {
                    push @return, $entry;
                }
            }

        } else {

            foreach my $record (sort keys %recordhash) {

                $infoline = sprintf("%-14s %-12s %-18s %-8s %-10s %-12s\n",
                                     $recordhash{$record}{NAME},
                                     $recordhash{$record}{ENTRY},
                                     $recordhash{$record}{TESTNAME},
                                     $recordhash{$record}{PLATFORM},
                                     $recordhash{$record}{TARGET},
                                     $recordhash{$record}{TABLE});
                push @return, $infoline;
            }
        }

        $entries = @return;

        if ($entries == 2) {
            @return = ();
            if (!$bHtml) {
                push @return, "No records found\n";
            }
        }

        _logevents("teststatus: $jobname $ServerName\n",0);

    } else {

        @return = _clientcall($ServerName, @argarray);

    }

    return @return;

}

sub runstats {

    @ARGV = @_;

    my %Options;

    my $ServerName;
    my $BuildName;
    my $ShowLast;
    my $AvgOnly;
    my $whoraw;
    my $recno     = 0;
    my $num;
    my $start;
    my $end;
    my $StartTime;
    my $EndTime;
    my $elapsed;
    my $avgtime;
    my $avginc = 0;
    my $avg;
    my $raw;
    my $xml;
    my $bHtml;
    my $startrec;
    my $printline;
    my $result;
    my $lastup;
    my $lastgoodelapsed = 0;
    my $bshowfailkey    = 0;
    my $bshowbtestkey   = 0;
    my $bshowgtestkey   = 0;

    my @ReturnArray;
    my @StatArray;

    my $ret;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:l:arxw', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $ShowLast   = $Options{l};
    $AvgOnly    = $Options{a};
    $whoraw     = $Options{r};
    $xml        = $Options{x};
    $bHtml      = $Options{w};

    #
    # For use with mysql
    #
    my $sqlquery;
    my @sqlarray;

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("runstats: $msghash{2}",1);
        return "missing job name\n"; 
    }

    if (($ret = _ValidJob($BuildName,$ServerName,0)) == 0) {
        _logevents("runstats: $BuildName $msghash{2}",1);
        return "undefined job $BuildName on $ServerName\n";
    }

    if (defined($AvgOnly)) {
        $avg = 1;
    }

    if (defined($whoraw)) {
        $raw = 1;
    }

    if (defined($xml)) {
        $xml = 1;
        $raw = 0;
        $avg = 0;
    }

    if ($ServerName =~ /^$hostname$/) {
        #
        # Just to make this more readable
        #
        $sqlquery = $qs->formatquery(44, $ServerName, $BuildName);
    
        @sqlarray = $os->run_sql_query("$sqlquery", ",", 0);
    
        foreach $sqlquery (@sqlarray) {
            push @StatArray, $sqlquery;
        }
    
        my $numrecs = @StatArray;
    
        if ($numrecs < 1) {
            _logevents("runstats: $BuildName no stats\n",1);
            @ReturnArray[0] = "no stats for $BuildName on $ServerName";
            return @ReturnArray;
        }
    
        if (defined($ShowLast)) {
    
            if ($ShowLast < 1) {
                $ShowLast = $ret;
            }
            if ($numrecs > $ShowLast) {
                $startrec = $numrecs - $ShowLast;
            } else {
                $startrec = 0;
            }
        } else {
            $ShowLast = $ret;
        }
    
        if ($numrecs > $ShowLast) {
            $startrec = $numrecs - $ShowLast;
        } else {
            $startrec = 0;
        }
    
        if (!$xml) {
            push @ReturnArray, "\n";
        } else {
            push @ReturnArray, "<build>\n";
        }
    
        if ((!defined($avg)) && (!defined($raw))) {
            $printline = sprintf("%-20s %-22s %-22s %-8s\n\n", "Job", "Start",
                "Finish", "Elapsed");
            push @ReturnArray, $printline;
        }
    
        for (; $startrec < $numrecs; $startrec++) {
    
            ($num, $start, $end, $result, $lastup) = 
                split(/,/, $StatArray[$startrec]);
    
            $StartTime = _gen_local_time_string($start);
            $EndTime   = _gen_local_time_string($end);
            $elapsed   = _format_elapsed_time($end - $start);
    
            if ($result != 2) {
                $avginc += ($end - $start);
            }
    
            if (!$xml) {
                if ((!defined($avg)) && (!defined($raw))) { if ($result == 0) {
                        $printline = sprintf("%-20s %-22s %-22s %-8s\n", $num,
                            $StartTime, $EndTime, $elapsed);
                    } elsif ($result == 1) {
                        $printline = sprintf("%-20s %-22s %-22s %-8s*\n", $num,
                            $StartTime, $EndTime, $elapsed);
                        $bshowbtestkey = 1;
                    } elsif ($result == 2) {
                        $printline = sprintf("%-20s %-22s %-22s %-8s**\n", $num,
                            $StartTime, $EndTime, $elapsed);
                        $bshowfailkey = 1;
                    } elsif ($result == 3) {
                        $printline = sprintf("%-20s %-22s %-22s %-8s***\n", $num,
                            $StartTime, $EndTime, $elapsed);
                        $bshowgtestkey = 1;
                    } 
                    push @ReturnArray, $printline;
                }
        
                if ($result != 2) {
                    $recno++;
                }
        
                if ((defined($raw)) && (!defined($avg))) {
                    if ($result != 2) {
                        if (!defined ($lastup)) {
                            $printline = sprintf("%s,%10d,%10d,%d,%d,%d\n", $num, 
                                $start, $end, $end-$start, $avginc/$recno, $result);
                        } else {
                            $printline = sprintf("%s,%10d,%10d,%d,%d,%d,%s\n", 
                                                 $num, $start, $end, $end-$start, 
                                                 $avginc/$recno, $result,
                                                 $lastup);
                        }

                        $lastgoodelapsed = $avginc/$recno;

                        push @ReturnArray, $printline;
                    } else {
                        if (!defined ($lastup)) {
                            $printline = sprintf("%s,%10d,%10d,%d,%d,%d\n", $num, 
                                                 $start, $end, $end-$start, 
                                                 $lastgoodelapsed, $result);
                        } else {
                            $printline = sprintf("%s,%10d,%10d,%d,%d,%d,%s\n", 
                                                 $num, $start, $end, $end-$start, 
                                                 $lastgoodelapsed, $result, 
                                                 $lastup);
                        }
                        push @ReturnArray, $printline;
                    }
                }
        
            } else {
                $printline = _xmlstats($BuildName, $num, $start, $end, $result);
                push @ReturnArray, $printline;
            }
        }
    
        if ($recno > 0) {
            $avgtime = _format_elapsed_time($avginc/$recno);
        } else {
            $avgtime = 0;
        }
    
        if (!defined($avg) && !$xml) {
            push @ReturnArray, "\n";
        }
    
        if ((defined($avg)) && (defined($raw)) && !$xml) {
            if ($recno > 0 ) {
                $printline = sprintf("%d", $avginc/$recno);
            } else {
                $printline = sprintf("%d", $recno);
            }
            push @ReturnArray, $printline;
        }
    
        if (!defined($raw) && !$xml) {
            if ($bshowbtestkey) {
                push @ReturnArray, "* build failed test\n";
            }
            if ($bshowfailkey) {
                push @ReturnArray, "** build failed to compile\n";
            }
            if ($bshowgtestkey) {
                push @ReturnArray, "*** build completed testing\n";
            }
            push @ReturnArray, "Average:  $avgtime\n";
        }

        if ($xml) {
            push @ReturnArray, "</build>\n";
        }
    
        _logevents("runstats: $BuildName\n",0);
    
    } else {
        my   @argarray;
        push @argarray, "runstats";
        push @argarray, "-n";
        push @argarray, "$BuildName";
        if (defined($ShowLast)) {
            push @argarray, "-l";
            push @argarray, "$ShowLast";
        }
        if ($AvgOnly) {
            $isexternal = 1;
            push @argarray, "-a";
        }
        if ($whoraw) {
            $isexternal = 1;
            push @argarray, "-r";
        }
        if ($xml) {
            $isexternal = 1;
            push @argarray, "-x";
        }
        if ($bHtml) {
            $isexternal = 1;
            push @argarray, "-w";
        }
        @ReturnArray = _clientcall($ServerName, @argarray);
        $isexternal = 0;
    }

    return @ReturnArray;
    
}

sub promote {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $JobNo;
    my $comment;
    my $promotetype;
    my $promotefrom;
    my $promoteto;
    my $promotesource;
    my $help;

    my %Options;

    my %promotestate;
    my $validpromote;

    my @argarray;
    my @retarray;

    my $commentline;
    my $fullcomment;

    my $delim      = $config->SMTPDELIM;
    my $promotecmd = $config->PROMOTECMD;
    
    my $args;
    my $ret;

    my $pid;
    my $osig;
    my $sig;
    my $recs;

    my $lreqtime        = $reqtime;
    my $lpeeraddress    = $peeraddress;
    $lreqtime           =~ s/\[//g;
    $lpeeraddress       =~ s/\]//g;

    #
    # For SQL...
    #
    my @SQLArray;
    my $sqlret;
    my $sqlquery;
    my $sqlline;
    my @queryarray;

    #
    # Mail configuration
    #
    my $ms        = $config->SMTP;
    my $admin     = $config->ADMIN;
    my $alternate = $config->ALTERNATE;
    my %mail;
    my %tohash;
    my $hashkey;

    #
    # Root dirs for source and targets as defined in buildconf.pm
    #
    my $source = $config->JOBDIR;
    my $target = $config->PROMOTION;

    $os->tracefunction((caller(0))[3]);

    getopts('S:t:f:n:j:c:h', \%Options);

    $ServerName  = $Options{S};
    $BuildName   = $Options{n};
    $JobNo       = $Options{j};
    $comment     = $Options{c};
    $promotetype = $Options{t};
    $promotefrom = $Options{f};
    $help        = $Options{h};

    if ($help) {

        %promotestate = _loadpromotions();

        if (!defined(%promotestate)) {
            _logevents("promote: $msghash{25}",1);
            return "no promotion.txt found on server\n";
        }

        foreach my $key (keys %promotestate) {
            $validpromote .= $key.", ";
        }
        $validpromote =~ s/, $//g;
        return "$validpromote are available for use\n";

    }
 
    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    #
    # Make sure there were args sent...
    #
    if (!defined($BuildName)) {
        _logevents("promote: $msghash{2}",1);
        return "missing job name\n";
    }

    #
    # Make sure there was a job number sent
    #
    if (!defined($JobNo)) {
        _logevents("promote: $msghash{9}",1);
        return "missing job number\n";
    }

    if (defined($promotefrom) && !defined($promotetype)) {
        _logevents("promote: $msghash{9}",1);
        return "cannot use -f without -t option\n";
    }

    if (defined($promotetype)) {

        %promotestate = _loadpromotions();

        if (!defined(%promotestate)) {
            _logevents("promote: $msghash{25}",1);
            return "no promotion.conf found on server\n";
        }

        $promoteto = $promotestate{$promotetype};

        if ($promoteto =~ /^$/) {
            foreach my $key (keys %promotestate) {
                $validpromote .= $key.", ";
            }
            $validpromote =~ s/, $//g;
                
            _logevents("promote: $msghash{9}",1);
            return "only $validpromote may be used with -t option\n";

        }

        if (defined($promotefrom)) {

            $promotesource = $promotestate{$promotefrom};

            if ($promotesource =~ /^$/) {
                foreach my $key (keys %promotestate) {
                    $validpromote .= $key.", ";
                }
                $validpromote =~ s/, $//g;
                    
                _logevents("promote: $msghash{9}",1);
                return "only $validpromote may be used with -f option\n";
    
            }

            if ($promoteto =~ /^$promotesource$/) {
                _logevents("promote: $msghash{9}",1);
                return "cannot promote to the same directory!\n";
            }

            if ($POSIX) {
                $source  = $target;
                $source .= "\/".$promotesource;
            } else {
                $source  = $target;
                $source .= "\/".$promotesource;
            }

        }
    } else {
        $promoteto = "promotion";
    }

    $promoteto =~ s/ //g;

    if ($POSIX) {
        $target .= "\/".$promoteto;
    } else {
        $target .= "\\".$promoteto;
    }

    #
    # Make sure this is a vaild job
    #
    if (!_ValidJob($BuildName,$hostname,0)) {
        _logevents("promote: $BuildName $msghash{2}",1);
        return "undefined job $BuildName\n";
    }

    #
    # Make sure there's a comment for the promotion
    #
    if (!defined($comment)) {
        _logevents("promote: $BuildName $msghash{9}",1);
        return "comment required for promote command\n";
    }

    $os->writelog("have all valid args to promote", *DEBUGGER);

    if ($ServerName =~ /^$hostname$/) {

        $os->writelog("Checking for freespace", *DEBUGGER);
        if ((_freespace($BuildName, 1)) == 1) {
            return "There is not enough disk space on $target for promotion\n";
        }

        $os->writelog("Verify that $source/$BuildName/$JobNo exists", 
                      *DEBUGGER);
        #
        # Make sure this job exists
        #
        if (! -d "$source/$BuildName/$JobNo") {
            _logevents("promote: $BuildName $msghash{18}",1);
            return "$BuildName $JobNo does not exist\n";
        }
    
        #
        $os->writelog("Verify that $target/$BuildName/$JobNo doesn't exist",
                      *DEBUGGER);
        #
        $sqlquery   = $qs->formatquery(114, $BuildName, $JobNo);
    
        @queryarray = $os->run_sql_query("$sqlquery", ";", 0);

        $recs = @queryarray;

        if ($recs) {
            _logevents("promote: $BuildName $JobNo $msghash{19}",1);
            return "$BuildName $JobNo already promoted in db\n";
        }

        # See if the job has already been promoted
        #
        if (-d "$target/$BuildName/$JobNo") {
            _logevents("promote: $BuildName $JobNo $msghash{19}",1);
            return "$BuildName $JobNo already promoted on fs\n";
        } else {
            $os->writelog("mkpath $target/$BuildName/$JobNo", *DEBUGGER);
            my $tomake = "$target/$BuildName/$JobNo";
            $tomake =~ s/\\/\//g;
            mkpath("$tomake", 0, 0755);
            if (! -d "$target/$BuildName/$JobNo") {
                $os->writelog("mkpath $target/$BuildName/$JobNo failed",
                             *DEBUGGER);
                _logevents("promote: $tomake $msghash{20}",1);
                return "failed to create target $target/$BuildName/$JobNo\n";
            }
            $os->writelog("mkpath completed", *DEBUGGER);
        }
    
        #
        # This sends out mail even before it completes. 
        #
        if ($POSIX) {
    
            $os->writelog("check for $promotecmd", *DEBUGGER);
            if ( -f "$promotecmd") {
                $os->writelog("fork $promotecmd", *DEBUGGER);
                $pid = $os->forkprocess(
                           "$promotecmd $source/$BuildName/$JobNo ".
                           "$target/$BuildName/$JobNo $BuildName $JobNo",
                           0, 0);
            } else {

                $os->writelog("fork cp command", *DEBUGGER);
                #
                # Call forkprocess and return with pid
                #
                $pid = $os->forkprocess(
                           "cp -r $source/$BuildName/$JobNo/* ".
                           "$target/$BuildName/$JobNo",
                           0, 0);

            }
    
        } else {
    
            if ( -f "$promotecmd") {
                $pid = $os->forkprocess(
                           "$promotecmd $source/$BuildName/$JobNo ".
                           "$target/$BuildName/$JobNo $BuildName $JobNo",
                           0, 0);
            } else {

                #
                # Spawn a separate process so that the buildserver can 
                # continue on...
                #
                $os->wincopyfiles("$source/$BuildName/$JobNo", 
                                  "$target/$BuildName/$JobNo");
            }
        
        }
    
        $os->writelog("Setup email for promote", *DEBUGGER);
        $mail{"Smtp"} = $ms;
        $mail{"From"} = "Build Promotion <$admin>";
        
        #
        # Gather up address and put them into a hash to eliminate
        # duplicate entries.
        #
        $sqlquery   = $qs->formatquery(10, $hostname, $BuildName);
    
        @queryarray = $os->run_sql_query("$sqlquery", ";", 0);
    
        foreach $sqlline(@queryarray) {
            chomp $sqlline;
            $tohash{"$sqlline"} = "";
        }
    
        $target =~ s/\\/\//g;
    
        $tohash{"$admin"} = "";
        $mail{Subject}    = "*$BuildName $JobNo promoted for testing*";
        $mail{Message}    = "$BuildName $JobNo was promoted to ".
                            "$target/$BuildName/$JobNo and is available ".
                            "for testing\n";
    
        foreach $hashkey (keys %tohash) {
            $mail{To} .= "$hashkey$delim";
        }
    
        sendmail %mail;
        
        push @SQLArray, "$hostname";
        push @SQLArray, "$BuildName";
        push @SQLArray, "$JobNo";
        push @SQLArray, "$lreqtime";
        push @SQLArray, "$lpeeraddress";
        push @SQLArray, "$comment";
    
        if (($ret = $os->run_sql_submit("promotion", @SQLArray)) == 0) {
            _logevents("promote: SQL failed $BuildName $JobNo\n",0);
            return "failed to add promoted job to SQL server\n";
        }

    } else {
 
        push @argarray, "promote";
        push @argarray, "-c";
        push @argarray, "\"$comment\"";
        push @argarray, "-n";
        push @argarray, "$BuildName";
        push @argarray, "-j";
        push @argarray, "$JobNo";

        if (defined($promotetype)) {
            push @argarray, "-t";
            push @argarray, $promotetype;
        }

        if (defined($promotefrom)) {
            push @argarray, "-f";
            push @argarray, $promotefrom;
        }

        @retarray = _clientcall($ServerName, @argarray);

        return @retarray;
    }

}

#
# Create a symlink to the latest tested build
#
sub passed {

    @ARGV = @_;

    my $BuildName;
    my $JobNo;
    my $comments;

    my @contents;
    my @numarray;
    my $nums;

    my $bCreateLink = 0;

    my $ret;

    my @cmdarray;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('n:j:c:', \%Options);

    $BuildName = $Options{n};
    $JobNo     = $Options{j};
    $comments  = $Options{c};

    if (!defined($BuildName)) {
        _logevents("passed: $msghash{2}",1);
        return "missing job name\n";
    }

    if (!defined($JobNo)) {
        _logevents("passed: $msghash{9}",1);
        return "missing job number\n";
    }

    if (!_ValidJob($BuildName,$hostname,0)) {
        _logevents("passed: $BuildName $msghash{2}",1);
        return "undefined job $BuildName\n";
    }

    if (! -d "$Moutdir/$BuildName/$JobNo") {
        _logevents("passed: $BuildName $JobNo $msghash{18}",1);
        return "$BuildName $JobNo does not exist\n";
    }

    @numarray = split(/\./, $JobNo);
    $nums = @numarray;
    $nums--;

    if (! -f "$Moutdir/.$BuildName.num") {
        open (TESTED, ">$Moutdir/.$BuildName.num");
        print TESTED "$numarray[$nums]";
        close(TESTED);
        $bCreateLink = 1;
    } else {
        open (TESTED, "<$Moutdir/.$BuildName.num");
        @contents = <TESTED>;
        close(TESTED);
        if ($numarray[$nums] > $contents[0]) {
            open (TESTED, ">$Moutdir/.$BuildName.num");
            print TESTED "$numarray[$nums]";
            close(TESTED);
            $bCreateLink = 1;
        }
    }
        

    if ($bCreateLink) {
        #
        # If there's a symlink, unlink it
        #
        if (-s "$Moutdir/$BuildName.tested") {
            print "$Moutdir/$BuildName.tested exists\n";
            unlink("$Moutdir/$BuildName.tested/.smokekeep") 
                || warn "Can't unlink file: $?";
            unlink("$Moutdir/$BuildName.tested") 
                || warn "Can't unlink file: $?";
        }
    
        #
        # Create a new symlink
        #
        chdir ("$Moutdir");
        symlink("$BuildName/$JobNo", 
                "$BuildName.tested") || warn "Can't create symlink: $?";
        chdir ("$dir");
    
        #
        # Arg list for keep
        #
        push @cmdarray, "-n";
        push @cmdarray, "$BuildName";
        push @cmdarray, "-j";
        push @cmdarray, "$JobNo";
        push @cmdarray, "-t";
        if (defined($comments)) {
            push @cmdarray, "-c"; 
            push @cmdarray, "$comments"; 
        }
    
        #
        # Run the keep command
        #
        $isinternal = 1;
        keep(@cmdarray);
        $isinternal = 0;
        
        #
        # Create a file to be used by external scripts if necessary
        # 
        open (KEEP, ">$Moutdir/.$BuildName.lasttested");
        print KEEP $JobNo;
        close(KEEP);
    
        #
        # Remove the default 'NOT TESTED' file then 
        # create a 'TEST PASSED' file
        #
        if (-f "$Moutdir/$BuildName/$JobNo/$nottested") {
            unlink ("$nottested");
            open (TESTED, ">$Moutdir/$BuildName/$JobNo/$tested");
            close(TESTED); 
        } else {
            open (TESTED, ">$Moutdir/$BuildName/$JobNo/$tested");
            close(TESTED); 
        }
    
    }

    _logevents("passed: $BuildName $JobNo\n",0);

    return $msghash{0};
    
}

sub keep {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $JobNo;
    my $comments;
    my $smoketest;

    my @rmtargs;
    my @cmdreturn;

    my %Options;

    my $ret;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:j:c:t', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $JobNo      = $Options{j};
    $comments   = $Options{c};

    #
    # Only process this option if it's internal.  We 
    # want to make sure that only smoketests make any 
    # alterations here.
    #
    if ($isinternal) {
        $smoketest  = $Options{t};
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("keep: $msghash{2}",1);
        return "missing job name\n";
    }

    if (!defined($JobNo)) {
        _logevents("keep: $msghash{9}",1);
        return "missing job number\n";
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("keep: $BuildName $msghash{2}",1);
        return "undefined job $BuildName\n";
    }

    if ($ServerName =~ /^$hostname$/) {
        if (! -d "$Moutdir/$BuildName/$JobNo") {
            _logevents("keep: $BuildName $JobNo $ServerName $msghash{18}",1);
            return "$BuildName $JobNo does not exist\n";
        }
        if (defined($smoketest)) {
            open (KEEP, ">$Moutdir/$BuildName/$JobNo/.smokekeep");
        } else {
            open (KEEP, ">$Moutdir/$BuildName/$JobNo/.keep");
        }
        print KEEP "$comments\n";
        close(KEEP);
        push @cmdreturn, $msghash{0};
    } else {
        push @rmtargs, "keep";
        push @rmtargs, "-n";
        push @rmtargs, "$BuildName";
        push @rmtargs, "-j";
        push @rmtargs, "$JobNo";
        if (defined($smoketest)) {
            push @rmtargs, "-t";
        }
        if (defined($comments)) {
            push @rmtargs, "-c";
            push @rmtargs, "$comments";
        }
        @cmdreturn = _clientcall($ServerName, @rmtargs);
    }

    _logevents("keep: $BuildName $JobNo $ServerName\n",0);

    return @cmdreturn;
    
}

sub cleanjob {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $JobNo;
    my $Force;
    my $Rollback;
    my $TestOnly;

    my @rmtargs;
    my @cmdreturn;

    my %Options;

    my $key;

    my @ret;

    my $sqlquery;
    my @sqlarray;

    my %tables = (
        "jobs"       => 0,
        "comments"   => 0,
        "changes"    => 0,
        "joblog"     => 0,
        "restore"    => 1,
        "semaphores" => 1,
        "restore"    => 1,
        "teststate"  => 2,
        "results"    => 2,
        "stage"      => 2,
    );

    my %testtables = (
        "run"       => 0,
        "teststate" => 0,
        "results"   => 0,
    );

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:r:j:t:f', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $JobNo      = $Options{j};
    $Force      = $Options{f};
    $Rollback   = $Options{r};
    $TestOnly   = $Options{t};

    if ($TestOnly && $Rollback) {
        _logevents("cleanjob: invalid args", 1);
        return "cannot rollback test tables\n";
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("cleanjob: $msghash{2}",1);
        return "missing job name\n";
    }

    if (!defined($JobNo)) {
        _logevents("cleanjob: $msghash{9}\n",1);
        return "missing job number\n";
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("cleanjob: $BuildName $msghash{2}",1);
        return "undefined job $BuildName\n";
    }

    if ($ServerName =~ /^$hostname$/) {

        if (defined($TestOnly)) {
            foreach $key (sort keys %testtables) {
                my $recs = 0;
                if (!$Force) {
                    if (!$testtables{$key}) {
                        $sqlquery = $qs->formatquery(32, $key, $BuildName, 
                                                      $JobNo, $TestOnly);
                        @sqlarray = $os->run_sql_query($sqlquery);
     
                        $recs = @sqlarray;
                        
                        if ($recs) {
                            push @ret, "would remove $recs record(s) from ".
                                       "$key table.\n";
                        } else {
                            push @ret, "no records found in table $key.\n";
                        }
                    }
                } else {
                    if (!$testtables{$key}) {
                        $sqlquery = $qs->formatquery(6, $key, $BuildName, 
                                                      $JobNo, $TestOnly);
                        $os->run_sql_remove("$sqlquery");
                        push @ret, "removed $TestOnly records from $key ".
                                   "table\n";
                    }
                }
            }
        } else {

            foreach $key (sort keys %tables) {
    
                my $recs = 0;
    
                if (!$Force) {
                    if (!$tables{$key}) {
                        $sqlquery = $qs->formatquery(33, $key, $BuildName, 
                                                      $JobNo);
                        @sqlarray = $os->run_sql_query($sqlquery);
     
                        $recs = @sqlarray;
                        
                        if ($recs) {
                            push @ret, "would remove $recs record(s) from ". 
                                       "$key table.\n";
                        } else {
                            push @ret, "no records found in table $key.\n";
                        }
                    } elsif ($tables{$key} == 1) {
                        $sqlquery = $qs->formatquery(34, $key, $BuildName, 
                                                      $JobNo);
                        @sqlarray = $os->run_sql_query($sqlquery);
     
                        $recs = @sqlarray;
                        
                        if ($recs) {
                            push @ret, "would remove $recs record(s) from ".
                                       "$key table.\n";
                        } else {
                            push @ret, "no records found in table $key.\n";
                        }
                    } elsif ($tables{$key} == 2) {
                        $sqlquery = $qs->formatquery(35, $key, $BuildName, 
                                                      $JobNo);
                        @sqlarray = $os->run_sql_query($sqlquery);
     
                        $recs = @sqlarray;
                        
                        if ($recs) {
                            push @ret, "would remove $recs record(s) from ".
                                       "$key table.\n";
                        } else {
                            push @ret, "no records found in table $key.\n";
                        }
                    }
     
                } else {
                    if (!$tables{$key}) {
                        $sqlquery = $qs->formatquery(7, $key, $BuildName, 
                                                      $JobNo);
                        $os->run_sql_remove("$sqlquery");
                    } elsif ($tables{$key} == 1) {
                        $sqlquery = $qs->formatquery(8, $key, $BuildName, 
                                                      $JobNo);
                        $os->run_sql_remove("$sqlquery");
                    } elsif ($tables{$key} == 2) {
                        $sqlquery = $qs->formatquery(9, $key, $BuildName, 
                                                      $JobNo);
                        $os->run_sql_remove("$sqlquery");
                    }
                    push @ret, "Records in table $key removed.\n";
    
                    if ($Rollback) {
    
                        my @rollback;
                        my $rollback;
                        my $command;
                        my @cmdarray;
    
                        #
                        # Retrieve charastics of the job being cleaned up
                        #
                        $sqlquery = $qs->formatquery(56, $BuildName, 
                                                      $hostname);
                        @sqlarray = $os->run_sql_query($sqlquery, ";");
    
                        my ($port, $client, $top, $sccs) = split(/;/, 
                                                           $sqlarray[0]);
    
                        $sqlquery = $qs->formatquery(40, $BuildName, 
                                                      $hostname, 1);
                        @sqlarray = $os->run_sql_query($sqlquery, ";");
    
                        if ( -f "$top/$BuildName.build") {
    
                            #
                            # Create an array by splitting up build number
                            #
                            my (@rollback) = split(/\./, $sqlarray[0]);
        
                            #
                            # Pop the last element off
                            #
                            my $lastbuild = pop @rollback;
        
                            my ($rollback) = join('.', @rollback);
    
                            #
                            # Roll back the major build number
                            #
                            open (BACK, "<$top/$BuildName.build");
                            my $num = <BACK>;
                            my $num--;
                            close(BACK);
                            
                            if ($rollback == $num) {
                                open (BACK, ">$top/$BuildName.build");
                                print BACK $num;
                                close(BACK);
                            } else {
                                push @ret, "Unable to rollback to $num build";
                            }
    
                        } else {
                            
                            $rollback = $sqlarray[0];
    
                        }
    
                        $command  = $sccs."_rollback";
                        @cmdarray = $cmbroker->$command($client, $port, $top, 
                                                        $rollback, $dir);
    
                        foreach my $one (@cmdarray) {
                            push @ret, $one;
                        }
                    }
    
                }
        
            }
        }

    } else {
        push @rmtargs, "cleanjob";
        push @rmtargs, "-n";
        push @rmtargs, "$BuildName";
        push @rmtargs, "-j";
        push @rmtargs, "$JobNo";
        if (defined($Force)) {
            push @rmtargs, "-f";
        }
        if ($Rollback) {
            push @rmtargs, "-r";
        }
        if ($TestOnly) {
            push @rmtargs, "-t";
        }
        @ret = _clientcall($ServerName, @rmtargs);
    }

    if (!$Force) {
        push @ret, "use -f to force removal of records.\n";
    }

    _logevents("cleanjob: $BuildName $JobNo $ServerName\n",0);

    return @ret;
    
}

sub free {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $JobNo;

    my %Options;

    my $ret;

    my @argarray;
    my @callarray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:j:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $JobNo      = $Options{j};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("free: $msghash{2}",1);
        return "missing job name\n";
    }

    if (!defined($JobNo)) {
        _logevents("free: $msghash{9}",1);
        return "missing job number\n";
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("free: $BuildName $msghash{2}",1);
        return "undefined job $BuildName\n";
    }

    if ($ServerName =~ /^$hostname$/) {

        if (! -d "$Moutdir/$BuildName/$JobNo") {
            _logevents("free: $BuildName $JobNo $msghash{18}",1);
            return "$BuildName $JobNo on $ServerName does not exist\n";
        }

        if (-f "$Moutdir/$BuildName/$JobNo/.keep") {
            unlink("$Moutdir/$BuildName/$JobNo/.keep");
        }

        _logevents("free: $ServerName $BuildName $JobNo\n",0);

        return $msghash{0};


    } else {
        push @argarray, "free";
        push @argarray, "-n";
        push @argarray, "$BuildName";
        push @argarray, "-j";
        push @argarray, "$JobNo";
        @callarray = _clientcall($ServerName, @argarray);
    } 

    _logevents("free: $ServerName $BuildName $JobNo\n",0);

    return @callarray;

}
    
sub display_command_usage {

    my $request = shift;
    
    my $help;

    $os->tracefunction((caller(0))[3]);

    _logevents("help $request\n",0);

    if (! defined $request) {
        $help = "help with what?";
    }
    else {
        $help = $supportedcmds{$request}{help};
        if (!defined($help)) {
            $help = $msghash{23};
        }

    }
    return $help;
}

sub display_commands {

    @ARGV = @_;

    my $blong = 0;
    my $html  = 0;

    my $c = 0;
    my @sc;
    my $i;
    my $left = 0;
    my $entries;
    my $leftside;
    my $rightside;
    my $cmdret = 0;
    my @returnArray;
    my @argarray;

    # stuff found by strict
    my $key;

    my $ServerName;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:lw', \%Options);

    $ServerName = $Options{S};
    $blong      = $Options{l};
    $html       = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname/) {

        if (!$blong) {
            $sc[$c] = "\n";
            $c++;
        } else {
            push @returnArray, "\n";
        }
    
        foreach $key (sort keys %supportedcmds) {
            if (!$blong) {
                $sc[$c] = "$key\n";
                $c++;
            } else {
                my $sr = $supportedcmds{$key}{help};
                if ($html) {
                    $sr =~ s/Usage/\<b\>$key\<\/b\>/g;
                } else {
                    $sr =~ s/Usage/$key/g;
                }
                push @returnArray, "\n$sr\n";
            }
        }
    
        if (!$blong) {
            $entries = @sc;
    
            $returnArray[$cmdret++] = "\n";
    
            for ($i = 0; $i < $entries; $i++) {
                chomp $sc[$i];
                if ($sc[$i] !~ /^$/) {
                    if ($left == 0) {
                        $leftside = $sc[$i];
                        $left++;
                    } else {
                        $rightside = $sc[$i];
                        $left = 0;
                        $returnArray[$cmdret++] = 
                            sprintf "%-15s %-15s\n", $leftside, $rightside;
                    }
                }
            }
        
            if ($left == 1) {
                sprintf "%-15s\n", $sc[$i-1];
                $returnArray[$cmdret] = 
                    sprintf "%-15s\n", $leftside;
            }
        
        }
    
        _logevents("commands\n",0);

    } else {
        if (defined($blong)) {
            push @argarray, "-l";
        }
        if (defined($html)) {
            push @argarray, "-w";
        }
        @returnArray = _clientcall($ServerName, "commands", @argarray);
    }

    push @returnArray, "\n";
    return @returnArray;

}
 
sub whosauthorized {

    @ARGV = @_;

    my @Contents;

    #
    # SQL stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $line;

    my $ServerName;
    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:',\%Options);

    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        push @Contents, "\n";
        push @Contents, "Computers authorized to control builds:\n\n";

        $sqlquery = $qs->formatquery(45, $hostname);

        @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);

        foreach $line (@sqlarray) {
            push @Contents, "\t$line\n";
        }

        push @Contents, "\n";

    } else {
        @Contents = _clientcall($ServerName, "authorized");
    }

    _logevents("authorized $ServerName\n",0);
    
    return @Contents;

}

sub authorize {

    @ARGV = @_;

    my $compname;
    my $ServerName;
    my $ret;

    my %Options;

    my $sqlquery;
    my @svrarray;
    my @retarray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:c:', \%Options);

    $ServerName = $Options{S};
    $compname   = $Options{c};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname/) {
        $sqlquery = $qs->formatquery(66, $ServerName);
        @svrarray = $os->run_sql_query($sqlquery, ";");
    
        $ret = @svrarray;
        
        if (!$ret) {
            _logevents("authorize $ServerName: $msghash{22}", 1);
            return "authorize $ServerName: $msghash{22}";
        }
     
        #
        # SQL Stuff...
        #
        my @sqlarray;
    
        if (!defined($compname)) {
            _logevents("authorize: $msghash{9}",1);
            return display_command_usage("authorize");
        }
    
        if (($ret = IsEntry($compname)) == 1) {
            _logevents("authorize $compname $ServerName: $msghash{12}",1);
            return "$msghash{12}";
        }
    
        push @sqlarray, $ServerName;
        push @sqlarray, $compname;
    
        if (($ret = $os->run_sql_submit("authtable", @sqlarray)) == 0) {
            _logevents("authorize $compname $ServerName: sql failed\n", 1);
        }

        _logevents("$compname authorized for build control\n",0);
        return "$compname authorized for buildcontrol\n";

    } else {
        @retarray = _clientcall($ServerName, "authorize", "-c", "$compname");
    }

    return @retarray;

}

sub unauthorize {

    @ARGV = @_;

    my $ServerName;
    my $compname;
    my $ret;

    my %Options;

    my @retarray;

    #
    # SQL Stuff...
    #
    my $sqlquery;

    $os->tracefunction((caller(0))[3]);

    getopts('S:c:', \%Options);

    $ServerName = $Options{S};
    $compname   = $Options{c};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ((!defined($compname))) {
        _logevents("unauthorize: $msghash{9}",1);
        return display_command_usage("unauthorize");
    }

    if ($ServerName =~ /^$hostname$/) {
        if (($ret = IsEntry($compname)) == 0) {
            _logevents("unauthorize $compname: $msghash{12}",1);
            return "$msghash{12}";
        }
    
        $sqlquery = $qs->formatquery(0, $hostname, $compname);
    
        if (($ret = $os->run_sql_remove("$sqlquery")) == 0) {
            _logevents("unauthorize $compname: sql failed", 1);
            return "SQL failed when trying to remove $compname from ".
                   "authtable\n";
        }
    
        _logevents("$compname unauthorized from $hostname build control\n",0);
        return "$compname unauthorized from build control on $hostname\n";
    } else { 
        @retarray = _clientcall($ServerName, "unauthorize", "-c", "$compname");
        return @retarray;
    }

}
    
sub subscribers {

    @ARGV = @_;

    my $BuildName;
    my $ServerName;
    my @Contents;
    my $ret;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;
    my $line;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("subscribers: $msghash{9}",1);
        return display_command_usage("subscribers");
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("subscribers $BuildName: $msghash{2}",1);
        return "$msghash{2}";
    }

    push @Contents, "\n";
    push @Contents, "Subscribers to $BuildName on $ServerName:\n\n";

    $sqlquery = formatquery(12, $ServerName, $BuildName);

    @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

    foreach $line (@sqlarray) {
        push @Contents, "\t$line\n";
    }

    push @Contents, "\n";

    _logevents("subscribers $BuildName on $ServerName\n",0);
    
    return @Contents;

}
    
sub unsubscribe {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $EmailAddr;
    my $ret;

    my %Options;

    #
    # SQL Stuff...
    #
    my $sqlstring;

    my @Contents;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:e:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $EmailAddr  = $Options{e};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ((!defined($EmailAddr)) || (!defined($BuildName))) {
        _logevents("unsubscribe: $msghash{9}",1);
        return display_command_usage("unsubscribe");
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("unsubscribe $ServerName $BuildName $msghash{2}",1);
        return "$msghash{2}";
    }

    if (($ret = OldAddress($ServerName, $BuildName,$EmailAddr)) == 0) {
        _logevents("unsubscribe $ServerName $BuildName $EmailAddr: ".
                   "$msghash{13}",1);
        return "$msghash{13}";
    }

    $sqlstring = $qs->formatquery(5, $ServerName, $BuildName, $EmailAddr);

    if (($ret = $os->run_sql_remove("$sqlstring")) == 0) {
        _logevents("unsubscribe $ServerName $BuildName $EmailAddr: ".
                   "SQL failed\n",1);
        return "SQL failed removing $EmailAddr from $BuildName\n";
    }
         
    _logevents("$EmailAddr unsubscribed from $ServerName $BuildName\n",0);
    return "$EmailAddr unsubscribed from $BuildName on $ServerName\n";

}

sub subscribe {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $EmailAddr;

    my %Options;

    my $ret;

    my %SubscribeHash;

    #
    # SQL Stuff...
    #
    my @SQLArray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:e:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $EmailAddr  = $Options{e};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ((!defined($EmailAddr)) || (!defined($BuildName))) {
        _logevents("subscribe: $msghash{9}",1);
        return display_command_usage("subscribe");
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("subscribe $BuildName: $msghash{2}",1);
        return "$msghash{2}";
    }

    if (($ret = OldAddress($ServerName, $BuildName,$EmailAddr)) == 1) {
        _logevents("subscribe $ServerName $BuildName $EmailAddr: ".
                   "$msghash{12}",1);
        return "$msghash{12}";
    }

    push @SQLArray, "$ServerName";
    push @SQLArray, "$BuildName";
    push @SQLArray, "$EmailAddr";

    if (( $ret = $os->run_sql_submit("subscription", @SQLArray)) == 0) {
        _logevents("subscribe $ServerName $BuildName $EmailAddr: ".
                   "SQL failed\n",1);
        return "Failed to write subscription info to SQL server\n";
    }

    _logevents("$EmailAddr subscribing to $ServerName $BuildName\n",0);
    return "$EmailAddr subscribed to $BuildName on $ServerName\n";

}

sub elapsed {

    @ARGV = @_;

    my $now;
    my $difference;
    my $seconds;
    my $minutes;
    my $hours;
    my $days;
    my $weeks;
    my $raw = 0;

    my $BuildName;
    my $rawarg;
    my $ServerName;

    my @TimeArray;
    my @GetRawNumbers;
    my $average;
    my $j;
    my $s;
    my $f,
    my $e;
    my $verbose;
    my $paverage;
    my $running = 0;
    my $entry;
    my $finishline;
    my $new = 0;
    my $intsethere = 0;

    my @contents;
    my @state;
    my @argarray;
    my @return;

    my @info;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $sqlline;
    my $sqlret;

    my %Options;

    # stuff found by strict
    my $printline;
    my $ret;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:rv', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $rawarg     = $Options{r};
    $verbose    = $Options{v};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($rawarg) {
        $raw = 1;
    }

    if (!defined($BuildName)) {
        _logevents("elapsed: $msghash{9}",1);
        return display_command_usage("elapsed");
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("elapsed $BuildName: $msghash{2}",1);
        return "invalid job: $BuildName\n";
    }

    if ($ServerName =~ /^$hostname$/) {

        _logevents("elapsed $BuildName\n",0);
    
        $sqlquery = $qs->formatquery(50, $ServerName, $BuildName, 1);
    
        @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
    
        if (@sqlarray > 0) {
            if ($sqlarray[0] !~ /build job disabled/) {
                $running = 1;
                @info = split(/ /, $sqlarray[0]);
            }
        } 

    
        if ($running == 1) {
    
            $sqlquery = $qs->formatquery(76, $ServerName, $BuildName, 1);
       
            @TimeArray = $os->run_sql_query("$sqlquery", ";", 0);
    
            $now = $os->_get_hosttime();
    
            if ($isinternal == 0) {
                $isinternal = 1;
                $intsethere = 1;
            }
    
            $paverage = $os->_get_average($ServerName, $BuildName);
    
            $difference = $now - $TimeArray[0];
            if (defined($paverage)) {
                if ($paverage > $difference) {
                    $finishline = _format_elapsed_time($paverage-$difference);
                } else {
                    $finishline = sprintf("may be overdue");
                }
            } else {
               $new = 1;
               $finishline = sprintf("no job data, may be a new job");
            }
    
            if ($raw != 1) {
                $seconds    =  $difference % 60;
                $difference = ($difference - $seconds) / 60;
                $minutes    =  $difference % 60;
                $difference = ($difference - $minutes) / 60;
                $hours      =  $difference % 24;
                $difference = ($difference - $hours)   / 24;
                $days       =  $difference % 7;
                $weeks      = ($difference - $days)    /  7;
                    
                if ($new == 0) {
                    $printline = sprintf("%s: %02d:%02d:%02d running, ".
                        "%s remaining",
                        $BuildName, $hours, $minutes,$seconds, $finishline);
                } else {
                    $printline = sprintf("%s: %02d:%02d:%02d running, ".
                        "%s",
                        $BuildName, $hours, $minutes,$seconds, $finishline);
                }
    
                _logevents("elapsed $BuildName\n",0);
            } else {
    
                if ($verbose) {
                    @state = _get_job_state($BuildName, 1);
                    $printline = sprintf("%s,%d,%d,%d,%d,%s", $info[1], 
                        $TimeArray[0], $now, $difference, $paverage, $state[0]);
                } else {
                    @state = _get_job_state($BuildName);
                    $printline = sprintf("%s,%d,%d,%d,%d,%d", $info[1], 
                        $TimeArray[0], $now, $difference, $paverage, $state[1]);
                }
    
            }
    
            if ($intsethere == 1) {
                $isinternal = 0;
            }
            return $printline;
    
        } else {
            return "$BuildName on $ServerName is not currently running\n";
        }
    } else {
        if (defined($BuildName)) {
            push @argarray, "-n";
            push @argarray, "$BuildName";
        }
        if (defined($rawarg)) {
            push @argarray, "-r";
        }
        @return = _clientcall($ServerName, "elapsed", @argarray);
        return @return;
    }
        
}
    
#
# Needs SQL...
#
sub build_status {

    @ARGV = @_;

    my $ServerName;
    my $title;
    my $bystep;
    my $line;
 
    my @line;

    my %Options;

    # SQL Stuff...
    my $sqlquery;
    my @sqlarray;
    my @sqlarray2;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:s', \%Options);

    $ServerName = $Options{S};
    $title      = $Options{n};
    $bystep     = $Options{s};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($Options{n})) {
        _logevents("status: $msghash{9}",1);
        return display_command_usage("status");
    }

    if ($ServerName =~ /^$hostname/) {

        $sqlquery = $qs->formatquery(68, $ServerName, $title);
    
        @sqlarray = $os->run_sql_query("$sqlquery", ",", 0);
         
        if (@sqlarray > 0) {
    
            if ($bystep) {
                $sqlquery = $qs->formatquery(73, $ServerName, $title, 1);
            } else {
                $sqlquery = $qs->formatquery(48, $ServerName, $title, 1);
            }
    
            @sqlarray2 = $os->run_sql_query("$sqlquery", ",", 0);
    
            if (@sqlarray2 > 0) {
                $line = $sqlarray2[0];
            }
    
        } else {
            $line = "$title does not exist on $ServerName";
        }
    
        _logevents("status $title\n",0);

        chomp $line;
    } else {
        @line = _clientcall($ServerName, "status", "-n", "$title");
        return @line;
    }

    return "$line\n";

}

sub describe_job {

    @ARGV = @_;

    my $BuildName;
    my $ServerName;
    my $Change;
    my $bHtml;
    my $bShift = 1;
    my $c = 0;

    my $Search;
    my $lastgood;
    my $rserver;
    my $rdepot;
    my $rdepotcounter;
    my $num;
    my $line;
    my $one;
    my $command;
    my @cmdarray;

    my @tmpbldarray;
    my @Darray;
    my @DepotArray;
    my @EmptyArray;
    my @rp4info;
    my @record;

    #
    # For SQL access...
    #
    my $sqlquery;
    my $returnstring1;
    my $returnstring2;
    my @sqlarray1;
    my @sqlarray2;
    my @JobArray;
    my @tmparray;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:c:w', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $Change     = $Options{c};
    $bHtml      = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    $Darray[0] = "no such job $BuildName on $ServerName";

    chomp $Darray[0];

    if (!defined($BuildName)) {
        _logevents("describe: msghash{9}",1);
        return display_command_usage("describe");
    }

    if ($ServerName =~ /^$hostname$/) {

        if ($BuildName =~ "all") {
            $Search = "[A-Z0-9]"; 
        } else {
            $Search = $BuildName;
        }
    
        #
        # Make query more readable
        #
        $sqlquery  = $qs->formatquery(83, $ServerName, 0);
    
        @sqlarray1 = $os->run_sql_query("$sqlquery", ";", 0);
    
        foreach $returnstring1 (@sqlarray1) {
    
            @tmparray = split(/;/, $returnstring1);
    
            #
            # No join because we could have instances with defined
            # jobs and no stats, so we'll run through separate 
            # queries...
            #
            $sqlquery  = $qs->formatquery(39, $tmparray[0], $ServerName, 1);
    
            @sqlarray2 = $os->run_sql_query("$sqlquery", ";", 0);
    
            foreach $returnstring2 (@sqlarray2) {
                $returnstring1 .= ";$returnstring2";
            }
    
            push @JobArray, $returnstring1; 
        }
    
        foreach $line (@JobArray) {
    
            my ($title,$port,$client,$top,$type,$tools,$isimake,
                $keep,$comment,$sccs,$cgiaccess,$lastjob) = split(/;/, $line);

            if ($title =~ /^$Search/i) {

                if ($bShift) {
                    @Darray = ();
                    $bShift = 0;
                }

                if ($bHtml) {

                    my %tablehash = ();
                    my @tabledata = ();

                    if (defined($lastjob)) {
                        $lastgood = $lastjob; 
                    } else {
                        $lastgood = "none";
                    }

                    %tablehash = (

                        1 => {
                            content => "$sccs",
                            class   => 'values',
                        },
                        2 => {
                            content => "$client",
                            class   => 'values',
                        },
                        3 => {
                            content => "$port",
                            class   => 'values',
                        },
                        4 => {
                            content => "$top",
                            class   => 'values',
                        },
                        5 => {
                            content => "$keep",
                            class   => 'values',
                        },
                        6 => {
                            content => "$lastgood",
                            class   => 'values',
                        },
                        7 => {
                            content => "$comment",
                            class   => 'values',
                        },
                    
                    );

                    @tabledata = tableData(%tablehash);
                      
                    foreach my $entry (@tabledata) {
                        push @Darray, $entry;
                    }

                } else {
                    if (defined($lastjob)) {
                        $lastgood = sprintf 
                                    "Last good build: $webserver/$title/%s\n", 
                                    $lastjob;
                    } else {
                        $lastgood = "Last good build: none\n";
                    }
        
                    $Darray[$c++] = " \n";
                    $Darray[$c++] = "Title: $title ($type)\n";
        
                    $command  = $sccs."_clientport";
                    if(defined($Change)) {
                        @cmdarray = $cmbroker->$command($client, $port, $Change);
                    } else {
                        @cmdarray = $cmbroker->$command($client, $port);
                    }
        
                    foreach $one (@cmdarray) {
                        $Darray[$c++] = $one;
                    }
        
                    @rp4info = "";
                                
                    $Darray[$c++] = "Build root: $top\n";
                    $Darray[$c++] = "Save level: $keep\n";
                    $Darray[$c++] = "$lastgood";
                    $Darray[$c++] = "Comment: $comment\n";
        
                }
            }
        }
    
        if (@Darray == 0) {
            return "ServerName: $msghash{14}";
        }
    
        _logevents("describe $ServerName $BuildName\n",0);

    } else {
        @Darray = _clientcall($ServerName, "describe", "-n", "$BuildName");
    }

    return @Darray;

}

sub reject_log {

    @ARGV = @_;

    my @Contents;
    my @TMPArray;
    my $result;
    my $elements;

    my $showlast;
    my $ServerName;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:l:', \%Options);

    $ServerName = $Options{S};
    $showlast   = $Options{l};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname/) {

        _logevents("admin rejectlog\n",0);
    
        if (-f "$dir/logs/$ServerName.errorlog") {
            open (LOGFILE, "<$dir/logs/$ServerName.errorlog") ||die "open: $!";
            push @Contents, "\n";
            push @Contents, "Reject log for $ServerName:\n\n";
            while (<LOGFILE>) {
                push @Contents, $_;
            }
            close (LOGFILE);
            my $elements = @Contents;
            if (defined($showlast)) {
                if ($elements > $showlast) {
                    $result = $elements - $showlast;
                    @TMPArray = splice(@Contents, $result, $showlast);
                    unshift(@TMPArray, "\n");
                    return @TMPArray;
                } else {
                    return @Contents;
                }
            }
            return @Contents;
        }

    } else {
        my @argarray;
        push @argarray, "rejectlog";
        if (defined($showlast)) {
            push @argarray, "-l";
            push @argarray, "$showlast";
        }
        @Contents = _clientcall($ServerName, @argarray); 
    }
   
    return @Contents;

}

sub connect_log {

    @ARGV = @_;

    my $ServerName;

    my @Contents;
    my @TMPArray;
    my $result;
    my $elements;

    my $showlast;

    my %Options;

    my @arglist;

    $os->tracefunction((caller(0))[3]);
 
    getopts('S:l:', \%Options);

    $ServerName = $Options{S};
    $showlast   = $Options{l};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($showlast)) {
        $showlast = 100;
    }

    _logevents("admin connectlog\n",0);

    if ($ServerName =~ /^$hostname$/) {

        if (-f "$dir/logs/$hostname.$logext") {
            open (LOGFILE, "<$dir/logs/$hostname.$logext") ||die "open: $!";
            push @Contents, "\n";
            push @Contents, "Connect log for $hostname:\n\n";
            while (<LOGFILE>) {
                push @Contents, $_;
            }
            close (LOGFILE);
            my $elements = @Contents;
            if (defined($showlast)) {
                if ($elements > $showlast) {
                    $result = $elements - $showlast;
                    @TMPArray = splice(@Contents, $result, $showlast);
                    unshift(@TMPArray, "\n");
                    return @TMPArray;
                } else {
                    return @Contents;
                }
            }
            return @Contents;
        }

    } else {
        push @arglist, "connectlog";
        if ($showlast) {
            push @arglist, "-l";
            push @arglist, "$showlast";
        }
        @Contents = _clientcall($ServerName, @arglist);
    }
    
    return @Contents;

}

sub my_connect_log {

    @ARGV = @_;

    my @Contents;
    my @TMPArray;
    my $result;
    my $elements;

    my $ServerName;

    my $showlast;

    my %Options;

    my @arglist;

    $os->tracefunction((caller(0))[3]);

    getopts('S:l:', \%Options);

    $ServerName = $Options{S};
    $showlast   = $Options{l};

    # stuff fund by strict
    my $x;

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($showlast)) {
        $showlast = 100; # Pick a default value
    }

    if ($ServerName =~ /^$hostname$/) {

        _logevents("user connectlog\n",0);

        if (-f "$dir/logs/$hostname.$logext") {
            open (OWNLOG, "<$dir/logs/$hostname.$logext");
            push @Contents, "\n";
            push @Contents, "Connect log for $hostname:\n\n";
            while (<OWNLOG>) {
                $x = index($_, $peername);
                if ($x>0) {
                    push @Contents, $_;
                }
            }
            close (OWNLOG);
            my $elements = @Contents;
            if (defined($showlast)) {
                if ($elements > $showlast) {
                    $result = $elements - $showlast;
                    @TMPArray = splice(@Contents, $result, $showlast);
                    unshift(@TMPArray, "\n");
                    return @TMPArray;
                } else {
                    return @Contents;
                }
            }
    
            return @Contents;
    
        }

    } else {
        push @arglist, "connectlog";
        if ($showlast) {
            push @arglist, "-l";
            push @arglist, "$showlast";
        }
        @Contents = _clientcall($ServerName, @arglist);
    }

    my $numhave = @Contents;
    
    if (!$numhave) {
        push @Contents, "no connectlog for $peername on $ServerName";
    }

    return @Contents

}

sub my_reject_log {

    @ARGV = @_;

    my @Contents;
    my @TMPArray;
    my $result;
    my $elements;

    my $ServerName;
    my $showlast;

    my %Options;

    # stuff found by strict
    my $x;

    $os->tracefunction((caller(0))[3]);

    getopts('S:l:', \%Options);

    $ServerName = $Options{S};
    $showlast   = $Options{l};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        _logevents("user rejectlog\n",0);

        if (-f "$dir/logs/$ServerName.errorlog") {
            open (OWNLOG, "<$dir/logs/$ServerName.errorlog");
            push @Contents, "\n";
            push @Contents, "Reject log for $ServerName:\n\n";
            while (<OWNLOG>) {
                $x = index($_, $peername);
                if ($x>0) {
                    push @Contents, $_;
                }
            }
            close (OWNLOG);
            my $elements = @Contents;
            if (defined($showlast)) {
                if ($elements > $showlast) {
                    $result = $elements - $showlast;
                    @TMPArray = splice(@Contents, $result, $showlast);
                    unshift(@TMPArray, "\n");
                    return @TMPArray;
                } else {
                    return @Contents;
                }
            }
    
            return @Contents;
    
        }

    } else {
        my @arglist;
        push @arglist, "rejectlog";
        if ($showlast) {
            push @arglist, "-l";
            push @arglist, "$showlast";
        }
        @Contents = _clientcall($ServerName, @arglist);
    }

    return @Contents

}

sub get_starts {

    @ARGV = @_;

    my @Contents;
    my @TMPArray;
    my $result;
    my $elements;

    my $ServerName;
    my $showlast;

    my @argarray;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:l:', \%Options);

    my $ServerName = $Options{S};
    my $showlast   = $Options{l};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        _logevents("laststart\n",0);

        if (-f "$dir/logs/$hostname.$logext") {
            push @Contents, "buildserver starts:\n\n";
            open (STARTLOG, "<$dir/logs/$hostname.$logext");
            while (<STARTLOG>) {
                if ($_ =~ /buildserver connectlog/) {
                    push @Contents, $_;
                }
            }
            close (STARTLOG);
            my $elements = @Contents;
            if (defined($showlast)) {
                if ($elements > $showlast) {
                    $result = $elements - $showlast;
                    @TMPArray = splice(@Contents, $result, $showlast);
                    unshift(@TMPArray, "\n");
                    return @TMPArray;
                } else {
                    return @Contents;
                }
            }
    
            return @Contents;
    
        }

    } else {
        push @argarray, "laststart";
        if ($showlast) {
            push @argarray, "-l";
            push @argarray, "$showlast";
        }
        @Contents = _clientcall($ServerName,  @argarray);
    }

    return @Contents

}

sub get_changes {

    @ARGV = @_;

    my @Contents;
    my @TMPArray;
    my @JobArray;
    my $client;
    my $user = "builder";
    my $port;
    my $result;
    my $elements;
    my $found;
    my $sccs;
    my $root;
    my $command;
    my $sccscommand;
    my $sccsclient;
    my $browser;

    my $ServerName;
    my $jobname;
    my $showlast;
    my $makeweb;

    my %Options;

    my $makeurl;

    # stuff found by strict
    my $ret;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;
    my $line;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:l:w', \%Options);

    $ServerName = $Options{S};
    $jobname    = $Options{n};
    $showlast   = $Options{l};
    $makeweb    = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($jobname)) {
        return display_command_usage("lastchange");
    }

    if (!_ValidJob($jobname,$ServerName,0)) {
        _logevents("lastchange $jobname\n",1);
        return "no such job $jobname on $ServerName\n";
    }

    if (!defined($showlast)) {
        $showlast = 10;
    }

    if ($ServerName =~ /^$hostname$/) {
        $sqlquery = $qs->formatquery(60, $jobname, $ServerName, 0);
    
        @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
    
        foreach $line (@sqlarray) {
            @JobArray =  split(/;/,$line);
            $found = 1;
        }
    
        $sccs       = $JobArray[0];
        $browser    = $JobArray[1];
        $root       = $JobArray[2];
        $sccsclient = $JobArray[3];
        $port       = $JobArray[4];
    
        $sccscommand = $sccs."_formatclientstring";

        $client      = $cmbroker->$sccscommand($sccsclient, $port);

        $found = 1;
    
        _logevents("lastchange $jobname\n",0);
    
        chomp $sccs;

        $command = $sccs."_lastcheckout";

        if (defined($makeweb)) {
            $makeurl = $browser;
        }

        @Contents = $cmbroker->$command($port, $client, $root, $jobname, 
                                        $showlast, $makeurl);

        unshift(@Contents, "\n");

    } else {

        my @cmdargs;

        push @cmdargs, "lastchange";
        push @cmdargs, "-n";
        push @cmdargs, "$jobname";
        if (defined($showlast)) {
            push @cmdargs, "-l";
            push @cmdargs, "$showlast";
        }
        if (defined($makeweb)) {
            push @cmdargs, "-w";
        }

        @Contents = _clientcall($ServerName, @cmdargs);
    }

    return @Contents

}

sub get_syncs {

    @ARGV = @_;

    my $ServerName;

    my @Contents;
    my @TMPArray;
    my @JobArray;
    my $client;
    my $user = "builder";
    my $port;
    my $result;
    my $elements;
    my $found;
    my $top;
    my $rserver;
    my $rclient;
    my $sccs;
    my @rp4array;
    my @rContents;
    my $command;

    my $jobname;

    my %Options;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $line;

    # stuff found by strict
    my $ret;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $jobname    = $Options{n};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($jobname)) {
        return display_command_usage("nextjob");
    }

    if (!_ValidJob($jobname,$ServerName,0)) {
        _logevents("nextjob $jobname\n",1);
        return "no such job $jobname\n";
    }

    if ($ServerName =~ /^$hostname$/) {

        $sqlquery = $qs->formatquery(55, $ServerName, $jobname);
        
        @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
        
        foreach $line (@sqlarray) {
            ($port, $client, $top, $sccs) = split(/;/, $line);
            $found = 1;
        }
    
        _logevents("nextjob $jobname\n",0);
        chomp $sccs;
    
        #
        # Use abstraction for stdoutupdate
        #
        $command = $sccs."_stdoutupdate";
        @Contents = $cmbroker->$command($jobname, $port, $client, $top, $dir);
        $os->writelog("$command $jobname, $port, $client, $top, $dir",
                     *DEBUGGER);

    } else {
        @Contents = _clientcall($ServerName, "nextjob", "-n", "$jobname");
    }

    return @Contents

}

sub _getbuildroot {

    my $BuildRoot;
    my @BuildInfo;

    my $SearchTitle = shift;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    $sqlquery  = $qs->formatquery(87, $hostname, $SearchTitle);

    @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);

    $BuildRoot = $sqlarray[0];

    return $BuildRoot

}

sub IsEntry {

    my $compname = shift;
    my $ret      = 0;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $line;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(46, $hostname, $compname);

    @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

    foreach $line (@sqlarray) {
        if ($compname =~ /^$line$/i) {
            $ret = 1;
        } 
    } 

    return $ret;

}

sub OldAddress {

    my $ServerName = shift;
    my $Job        = shift;
    my $Addr       = shift;
    my $ret        = 0;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(11, $ServerName, $Job, $Addr);

    @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

    $ret = @sqlarray;

    return $ret;

}

sub _ValidJob {

    my $BuildName  = shift;
    my $Server     = shift;
    my $SearchType = shift; 
    my @JobArray;
    my $x;
    my $c=0;

    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    #
    # Make query more readable
    #
    $sqlquery = $qs->formatquery(82, $Server, $BuildName, $SearchType);

    @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

    foreach $sqlquery (@sqlarray) {
        @JobArray = split(/;/, $sqlquery);
    }

    $c=$JobArray[7];

    return $c;
    
}

sub _logevents {

    my $msgs = shift;
    my $type = shift;

    chomp $msgs;

    $os->tracefunction((caller(0))[3]);

    if (!$isinternal) {

        if ($type == 0) {
            open ( CONNECTLOG, ">>$dir/logs/$hostname.$logext");
            if (defined($reqtime)) {
                print CONNECTLOG "$reqtime $peeraddress $msgs\n";
            } else {
                print CONNECTLOG "$msgs\n";
            }
            close(CONNECTLOG);
        } else {
            open ( ERRORLOG, ">>$dir/logs/$hostname.$errlogext");
            if (defined($reqtime)) {
                print ERRORLOG "$reqtime $peeraddress $msgs\n";
            } else {
                print ERRORLOG "$msgs\n";
            }
            close(ERRORLOG);
        }
    }

}

# Function to return a joblist, there can be no duplicates...
sub _getjoblist {

    my $ServerName = shift;

    my @jobName;
    my @Jobs;
    my $c = 0;

    # stuff found by strict
    my $key;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my $line;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(84, $ServerName);

    @sqlarray = $os->run_sql_query("$sqlquery", ";", "0");

    foreach $line (@sqlarray) {
        @jobName = split(/;/, $line);
        $jobHash{"$jobName[0]"} = "$jobName[1];$jobName[2]";
    }

    foreach $key (keys %jobHash) {
        $Jobs[$c] = $key;
        $c++;
    }

    return @Jobs;

}

sub _gen_xml_time_string {

    my $unixtime = shift;
    my $label    = shift;

    my $returnstring;
    
    my $format = '<%s year="%s" month="%02d" day="%02d" dayofweek="%s" '.
                 'dayofyear="%s" hour="%02d" minute="%02d">%s</%s>';

    $os->tracefunction((caller(0))[3]);

    if ($unixtime != 0) {
        my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = 
            localtime($unixtime);

        $returnstring = sprintf("$format", $label, 1900+$year, $mon+1, $mday,
                                 $wday, $yday, $hour, $min, $unixtime, 
                                 $label );
    } else {
        $returnstring = "<$label></$label>";
    }

    return $returnstring;

}

sub _gen_local_time_string {

    $os->tracefunction((caller(0))[3]);

    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($_[0]);

    my $returnstring = sprintf("%02d/%02d/%04d %02d:%02d:%02d", $mon+1, $mday,
        1900 + $year, $hour, $min, $sec);

    return $returnstring;

}

#
# Get the job state
#
sub _get_job_state {

    my $jobname = shift;
    my $verbose = shift;
    my @state;
    my @Contents;
    my $entries;
    my @Readit;
    my $buildnum;
    my $idx;
    my $jobline;
    my $message;

    $os->tracefunction((caller(0))[3]);

    $jobline = build_status("-n",$jobname, "-s");
    $message = build_status("-n",$jobname);

    $entries = @Contents;

    if ($verbose) {
        chomp $message;
        push @state, "$message";
        return @state;
    }
    
    if ($jobline == 0) {

        @Readit = split(/ /,$message);
        push @state, $Readit[2];
        push @state, "4";

    } elsif ($jobline == 1) {

        @Readit = split(/ /,$message);

        $idx = index($Readit[0], "\.");
        $buildnum = substr($Readit[0],$idx+1);

        push @state, $buildnum;
        push @state, "4";

    } elsif (($jobline == 2 ) ||
        ($jobline == 3) ||
        ($jobline == 4) ||
        ($jobline == 5)) {

        @Readit = split(/ /,$message);

        $idx = index($Readit[0], "\.");
        $buildnum = substr($Readit[0],$idx+1);

        push @state, $buildnum;
        push @state, "5";

    } elsif ($jobline == 6) {

        @Readit = split(/ /,$message);

        $idx = index($Readit[0], "\.");
        $buildnum = substr($Readit[0],$idx+1);

        push @state, $buildnum;
        push @state, "6";

    } elsif ($jobline == 7) {

        @Readit = split(/ /,$message);

        $idx = index($Readit[0], "\.");
        $buildnum = substr($Readit[0],$idx+1);

        push @state, $buildnum;
        push @state, "0";

    }

    return @state;

}

sub recover {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $ret;
    my $line;

    my %Options;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;
    my @sqlsubmit;

    my $return = "$BuildName cannot be recovered";
    my @return;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    
    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("recover $BuildName: $msghash{9}",1);
        return display_command_usage("recover");
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("restorejob $BuildName: $msghash{2}",1);
        return "$msghash{16}";
    }

    if ($ServerName =~ /^$hostname$/) {

        $sqlquery = $qs->formatquery(31, $hostname, $BuildName);

        @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
        if (@sqlarray > 0) {
            @sqlsubmit = split(/;/, $sqlarray[0]);
            if (($ret = $os->run_sql_submit("semaphores", @sqlsubmit)) == 0) {
                _logevents("restore: SQL failed\n",1);
                return "SQL failed to add record\n";
            }
            $return = "recovered $BuildName semaphore information";
        }

        _logevents("recover $BuildName\n",0);

    } else {
        @return = _clientcall($ServerName, "recover", "-n", "$BuildName");
        return @return;
    }
    
    return $return;

}

sub addjob {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $ret;
    my @iContents;
    my @oContents;
    my $a;
    my $line;
    my $return;
    my @return;

    my %Options;

    #
    # SQL Stuff...
    #
    my @sqlarray;
    my $sqlquery;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("restorejob $BuildName: $msghash{9}",1);
        return display_command_usage("restorejob");
    }

    if (_ValidJob($BuildName,$ServerName,0)) {
        _logevents("restorejob $BuildName: $msghash{17}",1);
        return "$msghash{17}";
    }

    if (!_ValidJob($BuildName,$ServerName,1)) {
        _logevents("restorejob $BuildName: $msghash{16}",1);
        return "$msghash{16}";
    }

    if ($ServerName =~ /^$hostname$/) {
    
        $sqlquery = $qs->formatquery(90, 0, $hostname, $BuildName);
    
        @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
        _logevents("restorejob $BuildName\n",0);
        
        $return = "job $BuildName restored";

    } else {
        @return = _clientcall($ServerName, "restorejob", "-n", "$BuildName");
        return @return;
    }
    
    return $return;

}

sub deljob {

    @ARGV = @_;

    my $ServerName;
    my $BuildName;
    my $Force;
    my $Found     = 0;
    my $ret;
    my @iContents;
    my @oContents;
    my $a;
    my $line;

    my %Options;

    #
    # For SQL...
    #
    my $sqlquery;
    my @sqlarray;
    my $sqlret;

    my $return;
    my @return;
    my @argarray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:f', \%Options);

    $ServerName = $Options{S};
    $BuildName  = $Options{n};
    $Force      = $Options{f};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($BuildName)) {
        _logevents("removejob $BuildName: $msghash{9}",1);
        return display_command_usage("removejob");
    }

    if (!_ValidJob($BuildName,$ServerName,0)) {
        _logevents("removejob $BuildName: $msghash{2}",1);
        return "$msghash{2}";
    }

    if ($ServerName =~ /^$hostname$/) {

        if ($Force) {
            $sqlquery = $qs->formatquery(1, $ServerName, $BuildName);
            $sqlret    = $os->run_sql_remove("$sqlquery");
        } else {
            $sqlquery = $qs->formatquery(90, 1, $ServerName, $BuildName);
            @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
        }
    
        _logevents("removejob $ServerName $BuildName\n",0);
        $return = "job $BuildName removed from $ServerName";

    } else {
        push @argarray, "removejob";
        push @argarray, "-n";
        push @argarray, "$BuildName";
        if ($Force) {
            push @argarray, "-f";
        }
        @return = _clientcall($ServerName, @argarray);
        return @return;
    }
    
    return $return;

}

sub show_removed {

    @ARGV = @_;

    my @options;

    my $ServerName;
    my $bHtml;

    my @contents;
    my @JobArray;
    my $line;
    my $bFound = 0;

    my %Options;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:w', \%Options);

    $ServerName = $Options{S};
    $bHtml      = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        if (!$bHtml) {
            push @contents, "\n";
            push @contents, "Jobs removed from buildserver control:\n\n";
        }
    
        $sqlquery = $qs->formatquery(80, $hostname, 1);
    
        @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
        
        foreach $line (@sqlarray) {
            $bFound = 1;
            if ($bHtml) {

                my %tablehash = ();
                my @tabledata = ();

                my $url;

                $url = "<a href=/adminbroker.php?cmd=restorejob&server=".
                       "$hostname&jobname=$line&sleep=1>$line</a>";

                %tablehash = (
                    1 => {
                        content => $url,
                        class   => 'values',
                    },
                );

                @tabledata = tableData(%tablehash);
                      
                foreach my $entry (@tabledata) {
                    push @contents, $entry;
                }
            } else {
                push @contents, "\t$line\n";
            }
        }
    
        _logevents("removed\n",0);
        
        if ($bFound == 1) {
            return @contents;
        } else {
            if ($bHtml) {
                return "";
            } else {
                return "no removed jobs present\n";
            }
        }

    } else {
        if ($bHtml) {
            push @options, "-w";
        } 
        @contents = _clientcall($ServerName, "removed", @options);
    }

}

#
# Email notification for a range of builds.  
#
sub notify {

    @ARGV = @_;

    # 
    # Command parameters
    #
    my $ServerName;
    my $job;
    my $start;
    my $end;
    my $msg;

    my %Options;

    my $line;
    my $subject;
    my $formatted;
    my @formattedinfo;
    my @info;

    my %tohash;
    my %mail;

    my $ret;
    my $return;
    my @return;

    my $Company   = $config->COMPANY;
    my $ms        = $config->SMTP;
    my $admin     = $config->ADMIN;
    my $delim     = $config->SMTPDELIM;
    my %to;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;
    my $length;
    my $cgiaccess;
    my $sccs;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:f:l:c:', \%Options);

    $ServerName = $Options{S};
    $job        = $Options{n};
    $start      = $Options{f};
    $end        = $Options{l};
    $msg        = $Options{c};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($job)) {
        _logevents("notify $job: $msghash{9}",1);
        return display_command_usage("notify");
    }

    if (!_ValidJob($job,$ServerName,0)) {
        _logevents("notify $job: $msghash{2}",1);
        return "$msghash{2}";
    }

    if (!defined($start)) {
        _logevents("notify $job start: $msghash{9}",1);
        return display_command_usage("notify");
    }

    if (!defined($end)) {
        _logevents("notify $job end: $msghash{9}",1);
        return display_command_usage("notify");
    }

    if (!defined($msg)) {
        _logevents("notify $job msg: $msghash{9}",1);
        return display_command_usage("notify");
    }

    if ($ServerName =~ /^$hostname$/) {

        #
        # Turn off logging since we'll use the 'changed' function
        #
        $isinternal = 1;
    
        @info = changed("-n", "$job", "-j", "$start", "-e", "$end");
    
        $isinternal = 0;
        
        $sqlquery = $qs->formatquery(10, $ServerName, $job);
    
        @sqlarray  = $os->run_sql_query("$sqlquery", ";", 0);
    
        $length = @sqlarray;
    
        foreach $line (@sqlarray) {
            chomp $line;
            $mail{Bcc} = "$line$delim";
        }
    
        $mail{Smtp}     = $ms;
        $mail{From}     = "Build Server <$admin>";
        $mail{Subject}  = "$msg";
    
        $mail{Message}  = "Changes for $job on $hostname from job $start ".
                          "to job\n $end:\n\n";
    
        foreach $line (sort @info) {
    
            my $command;
            my $emailaddress;

            my $formaturl;
    
            my ($who, $ver, $file, $sccs, $cgiaccess, $port) 
                = split(/,/,$line);
    
            #
            # Use abstraction for stdoutupdate
            #
            $command      = $sccs."_useraddress";
            $emailaddress = $cmbroker->$command($who, $port);
    
            $command   = $sccs."_formaturl";
            $formaturl = $cmbroker->$command($file, $ver, $cgiaccess, $port);
    
            if (!defined($emailaddress)) {
                $emailaddress = "$who\@$Company";
            }

            $to{"$emailaddress"} = "";
    
            $mail{Message} .= "change: $formaturl : $who\n";
            
        }
    
        foreach $line (keys %to) {
           $mail{Bcc} .= "$line$delim";
        }
    
        sendmail %mail;
    
        _logevents("notify $job\n",0);

        $return = "$subject sent";

    } else {

        @return = _clientcall($ServerName, "notify", "-n", "$job", "-f", 
                              "$start", "-l", "$end", "-c", "$msg");
        return @return;
    
    }

    return $return;
    
}
    
#
# Display files changed for a particular build
#
sub changed {

    @ARGV = @_;

    my $job;
    my $jobno;
    my $endjob;
    my $ServerName;
    my $bHtml;

    # my $format = shift;

    my $ret;

    my %Options;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:j:e:w', \%Options);

    $ServerName = $Options{S};
    $job        = $Options{n};
    $jobno      = $Options{j};
    $endjob     = $Options{e};
    $bHtml      = $Options{w};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($job)) {
        _logevents("changed $job: $msghash{9}",1);
        return display_command_usage("changed");
    }

    if (!_ValidJob($job,$ServerName,0)) {
        _logevents("changed $job $ServerName: $msghash{2}",1);
        return "$msghash{2}";
    }

    if (!defined($jobno)) {
        _logevents("changed $job $jobno $endjob: $msghash{9}",1);
        return display_command_usage("changed");
    }

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;
    my $length;
    my $line;
    my $formattedline;
    my $foundjob;
    my $foundchange;
    my @record;

    my @return;

    if ($ServerName =~ /^$hostname/) {
        #
        # Setup query as a range...
        #
        if (defined($endjob)) {
            $sqlquery = $qs->formatquery(18, $ServerName, $job, $jobno, 
                                          $endjob);
        } else {
            $sqlquery = $qs->formatquery(19, $ServerName, $job, $jobno);
        }
    
        @sqlarray  = $os->run_sql_query("$sqlquery", ",", 0);
        
        $length    = @sqlarray;
    
        if ($length) {
            
            if (!$isinternal && !$bHtml) {
                push @return, "\n";
                $formattedline = sprintf "%-21s %-12s %-10s %-55s\n", 
                                         "Job Number",
                                         "User", "Version", "File";
                push @return,  $formattedline;
                $formattedline = sprintf "%-21s %-12s %-10s %-55s\n", 
                                         "==========",
                                         "====", "=======", "====";
                push @return,  $formattedline;

            }
    
            foreach $line (@sqlarray) { 
                my $sccsquery;
                my @sccsarray;

                ($foundchange, $foundjob) = split(/,/,$line);

                @record = split(/;/, $foundchange);
    
                #
                # Print name, version, filename
                #
                if (!$isinternal && !$bHtml) {
                    chomp $record[0];
                    $formattedline = sprintf "%-21s %-12s %-10s %-55s\n", 
                        $foundjob, $record[3], $record[1], $record[0];
                } else {

                    $sccsquery = $qs->formatquery(59, $ServerName, $job, 
                                                  $foundjob);

                    @sccsarray = $os->run_sql_query($sccsquery, ";");

                    my ($sccs, $link, $port) = split(/;/, $sccsarray[0]);

                    if ($isinternal) {
                        $formattedline = sprintf "%s,%s,%s,%s,%s,%s", $record[3], 
                            $record[1], $record[0], $sccs, $link, $port;
                    } else {

                        my $record;
                        my $promoted;
                        my $email;
                        my $flink;
                        my $entry;
                        my %tablehash = ();
                        my @tabledata = ();

                        $formattedline = "";

                        my $cmd = $sccs."_formaturl";

                        $promoted = _ispromoted($job, $foundjob);

                        $record = 
                            $cmbroker->$cmd($record[0], $record[1], $link, 
                                            $port);

                        $email = _formatuseraddress($record[3], $sccs, $port);

                        $flink = makeLink("$record", "$record[0]");

                        %tablehash = (

                            1 => {
                                content => "$promoted",
                                class   => 'values',
                            },
                            2 => {
                                content => "$email",
                                class   => 'values',
                            },
                            3 => {
                                content => "$record[1]",
                                class   => 'values',
                            },
                            4 => {
                                content => "$flink",
                                class   => 'values',
                            },
                        
                        );

                        @tabledata = tableData(%tablehash);
                        
                        foreach $entry (@tabledata) {
                            $formattedline .= $entry;
                        }

                    }

                } 

                push @return, "$formattedline";
    
            }
            if (!$isinternal) {
                push @return, "\n";
            }
    
        } else {
            push @return, "no records found!";
        }
    
        _logevents("changed $ServerName $job $jobno $endjob\n",0);
    
    } else {
        my @argarray;
        push @argarray, "changed";
        push @argarray, "-n";
        push @argarray, "$job";
        if (defined($jobno)) {
            push @argarray, "-j";
            push @argarray, "$jobno";
        }
        if (defined($endjob)) {
            push @argarray, "-e";
            push @argarray, "$endjob";
        }
        if (defined($bHtml)) {
            push @argarray, "-w";
        }

        @return = _clientcall($ServerName, @argarray);
    }

    return @return;
    
}

#
# Build Server HW/SW combination
#
sub sysinfo {

    @ARGV = @_;

    my $ServerName;
    my %Options;

    my @retarray;

    $os->tracefunction((caller(0))[3]);

    getopts('S:', \%Options);

    $ServerName = $Options{S};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    _logevents("sysinfo\n",0);

    if ($ServerName =~ /^$hostname$/) {
        return $os->sysinfo();
    } else {
        @retarray = _clientcall($ServerName, "sysinfo");
    }

    return @retarray;

}
    
#
# FIXFIX - need to abstract some of these calls to support CVS
#
sub createjob {

    @ARGV = @_;

    my %opts;
    my $validopt;
    my $bMissing = 0;
    my $bGood = 0;
    my $sGood = 0;
    my $ret;
    my $sccscommand;

    my $ServerName;
    my $dump = 0;
    my $buildname;
    my $port;
    my $client;
    my $top;
    my $type;
    my $tools;
    my $isimake = "no";
    my $keep;
    my $comment;
    my $sccs;
    my $browser;
    my $state = 0;
    my $spam;
    my $size = 1;

    #
    # SQL Stuff...
    #
    my @SQLArray;
    my $sqlquery;
    my @sqlarray;

    my @errormsg;

    my @supported = qw ( perforce cvs subversion);

    my @options = qw (n p c r t d k C s b m);
    my @types   = qw (retail debug both);

    my $invalidchars = 
        "#\\s\\t;:\\<\\>,\\.\\?\\/\\'\\\"\\[\\]\\{\\}\\\\\\|!-()";
    my $dinvalidchars = 
        "#\\s\\t;\\<\\>,\\?\\/\\'\\\"\\[\\]\\{\\}\\|!()";

    # OS SPECIFIC
    my @initfiles;

    if ($POSIX) {
        @initfiles = qw (prebuild.sh postbuild.sh postpostbuild.sh onfail.sh);
    } else {
        @initfiles = qw (prebuild.cmd postbuild.cmd postpostbuild.cmd onfail.cmd);
    }

    $os->tracefunction((caller(0))[3]);

    getopts("S:n:p:c:r:t:d:k:C:s:b:m:D", \%opts);

    $ServerName = $opts{S};
    $buildname  = $opts{n};
    $port       = $opts{p};
    $client     = $opts{c};
    $top        = $opts{r};
    $type       = $opts{t};
    $tools      = $opts{d};
    $keep       = $opts{k};
    $comment    = $opts{C};
    $sccs       = $opts{s};
    $browser    = $opts{b};
    $spam       = $opts{m};
    $dump       = $opts{D};

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if (!defined($buildname)) {
        return "missing job name\n";
    }

    if ($dump) {

        if (!_ValidJob($buildname,$ServerName,0)) {
            _logevents("createjob dumper nonexistant job $buildname\n",1);
            return "job $buildname on $ServerName undefined\n";
        }

        $sqlquery = $qs->formatquery(28, $ServerName, $buildname, 1);

        @sqlarray = $os->run_sql_query($sqlquery, ",");

        ($ServerName,$buildname,$port,$client,$top,$type,$tools,
         $isimake,$keep,$comment,$sccs,$browser,$state,$spam,
         $size) = split(/,/, $sqlarray[0]);

        return "createjob -n $buildname -p $port -c $client -r $top ".
               "-t $type -d $tools -k $keep -s $sccs -b $browser ".
               "-m $spam -C \"$comment\"\n";

    } 

    push @errormsg, "\n";

    foreach $validopt (@options) {
        if (!defined($opts{$validopt})) {
            if ($bMissing == 0) {
                print "Incomplete or missing argument: ";
                $bMissing = 1;
            } 
            push @errormsg, "missing arg -$validopt\n";
        }
    }

    if ($bMissing == 1) {
        _logevents("createjob missing option\n",1);
        # return display_command_usage("createjob");
        return @errormsg;
    }

    if ($buildname =~ /[$invalidchars]/) {
        _logevents("createjob invaild jobname\n",1);
        return "invalid characters in job name\n";
    }

    #
    # Look for supported SCCS system...
    #
    foreach my $tmplook (@supported) {
        if ($opts{s} =~ /^$tmplook$/i) {
            $sGood = 1;
        } 
    }

    if (!$sGood) {
        _logevents("createjob invaild sccs\n",1);
        return "invalid sccs specified\n";
    }
        
    foreach $validopt (@types) {
        if ($type =~ /^$validopt$/i) {
            $bGood = 1;
        } 
    }

    if (!$bGood) {
        _logevents("createjob invaild type\n",1);
        return "invalid type specified\n";
    }

    #
    # Validate the port for the CM system
    #
    $sccscommand = $sccs."_verifyport";

    print STDERR "about to run $sccscommand with arg of $port\n";

    #
    # Call the broker
    #
    $ret = $cmbroker->$sccscommand($port);

    if ($ret) {
        _logevents("createjob invaild port\n",1);
        return "invalid port specified!\n";
    }

    if ($client =~ /[$invalidchars]/ && $sccs !~ /cvs/i ) {
        _logevents("createjob invalid client name\n",1);
        return "invalid characters in client name\n";
    }

    if (!$POSIX) {
        $top =~ s/\//\\/g;
    }

    if (!$POSIX) {
        if (($top !~ /^[A-Za-z]:\\\\[A-Za-z0-9]+/) || 
            ($top =~ /[$dinvalidchars]/)) {
            _logevents("createjob invaild root spec\n",1);
            return "invalid characters in root specification\n";
        }
    } else {
        if ($top !~ /^\//) {
            _logevents("createjob invaild root spec\n",1);
            return "invalid characters in root specification\n";
        }
    }

    if (!$POSIX) {
        $tools =~ s/\//\\/g;
        if (($tools !~ /^[A-Za-z]:\\[A-Za-z0-9]+/) || 
            ($tools =~ /[$dinvalidchars]/)) {
            _logevents("createjob invaild tool spec\n",1);
            return "invalid characters in tools specification\n";
        }
    } else {
        if ($tools !~ /^\//) {
            _logevents("createjob invaild tools spec\n",1);
            return "invalid characters in tools specification\n";
        }
    }

    if ($keep !~ /^[0-9]+$/) {
        _logevents("createjob invaild keep type\n",1);
        return "non-numeric keep level specified\n";
    }

    if (_ValidJob($buildname,$ServerName,0)) {
        _logevents("createjob job exists\n",1);
        return "job already defined and under build control\n";
    }

    if (_ValidJob($buildname,$ServerName,1)) {
        _logevents("createjob job exists\n",1);
        return "job defined but not under build control\n";
    }
    
    push @SQLArray, "$ServerName";
    push @SQLArray, "$buildname";
    push @SQLArray, "$port";
    push @SQLArray, "$client";
    push @SQLArray, "$top";
    push @SQLArray, "$type";
    push @SQLArray, "$tools";
    push @SQLArray, "$isimake";
    push @SQLArray, "$keep";
    push @SQLArray, "$comment";
    push @SQLArray, "$sccs";
    push @SQLArray, "$browser";
    push @SQLArray, "0";
    push @SQLArray, "$spam";
    push @SQLArray, "$size";

    if (($ret = $os->run_sql_submit("configuration", @SQLArray)) == 0) {
        _logevents("createjob: SQL failed \n",1);
        return "SQL failed to add record\n";
    }

    if (defined($opts{f})) {
        if (! -d $top) {
            open (CMD, "mkdir $top|") || die "$!: mkdir $opts{r}";
            close (CMD);
        }
        for my $file (@initfiles) {
            open (IN, "<$dir/skeleton/$file") || die "$!: open in";
            my @contents = <IN>;
            close (IN);
            open (OUT, ">$top/$file") || die "$!: open out";
            print OUT @contents;
            close(OUT);
        }
    }

    _logevents("createjob $opts{n}\n",0);
    return "job $buildname created on $ServerName\n";

}

sub genweb {

    @ARGV = @_;

    # 
    # Setup some global variables
    #
    my $one;
    my $c;
    my @temparray;
    my @tmparray;
    my @jobs;
    my $templine;
    my @TmpJob;
    my $jj;

    my $ServerName;
    my $JobName;

    my %Options;

    $viewitems = 0;

    $os->tracefunction((caller(0))[3]);

    getopts('S:n:l:', \%Options);

    $ServerName = $Options{S};
    $JobName    = $Options{n};
    $viewitems  = $Options{l};

    # 
    # Initialize global variables
    #
    %mainhash     = ();
    %jobhash      = ();
    @jobarray     = ();
    @stats        = ();
    %idlehash     = ();
    %statehash    = ();
    @reversearray = ();
    @timearray    = ();
    %workhash     = ();
    @webarray     = ();

    # stuff found by strict
    my $entry;
    my $pass;
    my $showit;
    my $key;
    my $doskip;
    my $futurestate;
    my $printstring;
    my $xxx;
    my $zzz;
    my $xzxz;
    my $job;
    my $line;
    my @TheseJobs;

    my $g_all = 0;

    #
    # Get the number of jobs to display
    #
    if (!defined($JobName) || !($viewitems)) {
        $isinternal = 0;
        _logevents("genweb $JobName $viewitems: $msghash{7}",1);
        return display_command_usage("genweb");
    }

    if (!defined($ServerName)) {
        $ServerName = $hostname;
    }

    if ($ServerName =~ /^$hostname$/) {

        $isinternal = 1;
    
        _getjoblist($ServerName);
    
        $itsnow = $os->_get_hosttime();
        # $itsnow = time();
        
        my @SELPLATFORM;
        my @JOBS;
        
        $HASARGS = 0;
        
        #
        # Did we get an arg from the command line?
        #
        if ($JobName =~ /^all$/) {
        
            @temparray = describe_job('-n', 'all');
    
            foreach $templine (@temparray) {
                if ($templine =~ /Title\:/) {
                    push @TheseJobs, $templine;
                }
            }
        
            unshift(@TheseJobs, "all");
        
            foreach $job (@TheseJobs) {
                if ($job ne "all") {
                    push @jobs, $job;
                }
            }
        
            $HASARGS = 1;
        
        } else {
        
            my @TJobs = split(/ /, $JobName);
    
            foreach my $_job (@TJobs) {
        
                @temparray = describe_job('-n', "$_job");
            
                foreach $templine (@temparray) {
                    if ($templine =~ /Title\:/) {
                        push @TmpJob, $templine;
                    }
                }
    
                foreach $entry (@TmpJob) {
                    push @jobs, $entry;
                }
            }
        
            $HASARGS = 1;
        
        }
    
        #
        # Get jobs into a hash table
        #
        %jobhash = _getjobnames(@jobs);
        %idlehash = %jobhash;
        %statehash = %jobhash;
        %workhash;
        
        # Initialize Job description arrays...
        foreach $one (keys %jobhash) {
            @$one = describe_job('-n', "$one");
        }
        
        #
        # Read through the hash table...
        #
        foreach $key (sort keys %jobhash) {

            my $mainaverage;
        
            push @jobarray, $key;
        
            #
            # Get stats for each job...
            #
            @stats = runstats("-n", $key,"-l" ,$viewitems, "-r");
            $mainaverage = $os->_get_average($ServerName, $key);
        
            _get_running_jobs($key);
        
            #
            # Read stats into the hash, this will take care of jobs fired off
            # at exactly the same time.
            # 
            foreach $line (@stats) {
        
                #
                # Remove newline
                #
                chomp $line;
        
                if ($line ne " ")  {
        
                    #
                    # Generate an array from each line.
                    # 
                    @tmparray = split (/,/, $line);
        
                    #
                    # Ignore bogus entries...
                    #
                    if ($tmparray[1] ne "" ) {
            
                        #
                        # Get rid of space in job number...
                        #
                        $tmparray[0] =~ s/ //g;
            
                        #
                        # Add entry to hash table...
                        #
                        %{ $mainhash{$tmparray[1]}{$key} } = ( 
                            jobno   => $tmparray[0],
                            length  => $tmparray[3],
                            finish  => $tmparray[2],
                            avg     => $mainaverage,
                            status  => $tmparray[5],
                            lastcg  => $tmparray[6],
                        );
    
                        if (defined($tmparray[2])) {
            
                            foreach $jj (keys %jobhash) {
                                %{ $mainhash{$tmparray[2]}{$jj} } = (
                                    status  => 99,
                                );
                            }
            
                            if (defined
                               ($mainhash{$tmparray[2]}{$key}{status})) {
                                delete($mainhash{$tmparray[2]}{$key}{status});
                            }
    
    
                            %{ $mainhash{$tmparray[2]}{$key} } = (
                               $tmparray[1], 
                            );
            
                        }
            
                        foreach $jj (keys %jobhash) {
            
                            if (!defined
                               ($mainhash{$tmparray[1]}{$jj}{jobno})) {
                                %{ $mainhash{$tmparray[1]}{$jj} } = (
                                    status  => 99,
                                ); 
                            }
                        }
            
                    }
                }
            }
        
        }
        
        _genheader();
        
        foreach $job (keys %jobhash) {
            $statehash{$job} = 0;
        }
        
        foreach $key (sort keys %mainhash) {
            push @timearray, $key;
        }
        
        $pass = 1;
        
        #
        # Print entries from sorted hash table (sorted by start)
        #
        foreach $key (sort keys %mainhash) {
    
        
            #
            # Increment counters for all jobs, set statehash
            #
            foreach $job (keys %jobhash) {
                $jobhash{$job}++;
                $idlehash{$job}++;
            }
        
            #
            # Print start time (main entry point into hash)
            #
        
            push @reversearray, "</tr>\n";
        
            #
            # Since there's another has here, read through it...
            #
            $a = 1;
            foreach $entry (reverse sort keys %{ $mainhash{$key} }) {
    
                
                # 
                # This should print out jobname...
                #
        
                $zzz = 0;
                foreach $xzxz (keys %{ $mainhash{$key}{$entry}}) {
                    $zzz++;
                }
        
                if (defined($mainhash{$key}{$entry}{finish})) {
                    $idlehash{$entry} = 1;
                    $statehash{$entry} = 1;
                    $xxx = $mainhash{$key}{$entry}{finish};
                    %{ $mainhash{$xxx}{$entry}} = (
                        $key,
                    );
                }
        
    
                # 
                # Here's where the nasty stuff starts
                #
                foreach $one (keys %{ $mainhash{$key}{$entry} } ) {
    
                   
                    # 
                    # We found the start of the build, time to print 
                    # skipline info
                    #
                    if (!defined($mainhash{$key}{$entry}{status}))  {
    
        
                        $printstring = _gendisplaystring(@$entry,$one);
                        #
                        # Push this onto the array
                        #
                        push @reversearray, 
                            "<td align=center bgColor=".
                            "$colorarray[$mainhash{$one}{$entry}{status}] ".
                            "rowspan=$idlehash{$entry}>$printstring</td>\n";
                        #
                        # Reset build states (we become idle)
                        #
                        $idlehash{$entry} = -1;
                        $statehash{$entry} = 0;
        
                    }
        
                    if (defined($mainhash{$key}{$entry}{$one})) {
                        # 
                        # If this isn't a 'filler' (status = 99)
                        #   
                        if ($mainhash{$key}{$entry}{$one} != 99) {
                            $doskip = 1;
                        } elsif ($statehash{$entry} == 0) {
                            #
                            # Use entries from the array to look at the 
                            # next entry to see if this job fires off.
                            #
                            $futurestate = _inthefuture($pass, $entry);
        
                            #
                            # The next time entry has a start for this job, 
                            # print skipline info for the accumulated idle 
                            # entries
                            #
                            if ($futurestate == 1) {
                                $idlehash{$entry}++;
                                push @reversearray, "<td bgColor=#ffffc8 ".
                                                  "rowspan=$idlehash{$entry}>".
                                                  "&nbsp;</td>\n";
                            } else {
                                ; 
                            }
                        } 
                    } else {
        
                        #
                        # This is the end of the build, set the 
                        # skipcounter to 0
                        #
                        $jobhash{$entry} = 0;
                        $jobhash{$entry} = 0;
                    }
                }
        
                #
                # Here we reset the skipline counter to 0
                #
                if ($doskip == 1) {
        
                    #
                    # Print skipline, reset counter to 0...
                    #
                    $jobhash{$entry} = 0;
                    $doskip = 0;
                }
        
                $a++;
        
            }
        
            $showit = _gen_time_string($key,0);
        
            push @reversearray, "<td nowrap bgColor=#dddddd>$showit</td>\n";
            push @reversearray, "<tr>\n";
        
        
            $pass++;
        
        }
        
        foreach $entry (reverse @reversearray) {
            push @webarray, $entry;
        }
    
        
        push @webarray, "</tbody>";
        push @webarray, "</table>";
    
        $isinternal = 0;

        _logevents("genweb @_\n",0);

    } else {
        @webarray = _clientcall($ServerName, "genweb", "-n", "$JobName", 
                                "-l", "$viewitems");
    }

    return @webarray;

}

#
# Function to read jobs into a hash table
#
sub _getjobnames {
    
    my $entry;
    my @details;
    my %return;

    my $c;

    $os->tracefunction((caller(0))[3]);

    foreach $entry (@_) {

        @details = split(/ /, $entry);
        
        # 
        # We're going to use this as the skipline counter...
        #
        $return{$details[1]} = 0;

    }

    return %return;

}

sub _genheader {

    my $numjobs = @jobarray;
    my @statusfile;
    my $line;
    my $printline;
    my $disabled;

    my $IDLEMSG = "Job Idle";
    
    # Stuff found by strict
    my $entry;
    my $hereandnow;
    my $mm;

    my $jc = 0;

    my $disicon    = $config->DISICON;
    my $logoicon   = $config->LOGOICON;
    my $webroot    = $config->WEBROOT;
    my $infoicon   = $webroot."/icons/info2.gif";
    my ($desc, $port, $stat);;

    my $sqlquery;
    my @sqlarray;

    my $product    = "Continuous Automated Build and Integration Environment";

    $os->tracefunction((caller(0))[3]);

push @webarray, <<EOF;
<html>
<head>
<link rel="SHORTCUT ICON" HREF="/icons/cabie.gif">
<link rel="stylesheet" href="$webroot/main/static/style/bims.css" type="text/css">
<link rel="alternate" title="CABIE RSS" href="$cgibin/cabie.rss" type="application/rss+xml">
<!-- #BeginEditable "head" -->
<!-- #EndEditable -->
<style type="text/css">
<--
<--
body{
overflow:scroll;overflow-x:hidden
}
.bulle
{
  position : absolute;
  visibility : hidden;
}
//-->
</style>
<title>$product</title>
</head>
<body bgcolor="#f0f0f0">
<div class=\"bulle\" id=\"topdeck\"></div>

<script>

var ns = (document.layers); 
var ie = (document.all);
var ns7 = (!document.all && document.getElementById);

initialized="0";

if (ns7)
{
    var doc = document.getElementById("topdeck");
}

var skn = (ns || ns7) ? document.topdeck : topdeck.style;

function loadpopup(web,server,title,job)
{
    var line = web+"?srvr="+server+"&title="+title+"&startjob="+job;
    popupwin = window.open(line,"!I(Server)","width=300,height=310,location=no,toolbar=no,menubar=no,scrollbars=no,resizable=no")
}

function nothing()
{
    initialized = 0;
}

function pop(texte,fond,title)
{
    var msg  = "";
    var ttle = "";
    var line = "";
    msg      = texte;
    ttle     = title;
    initialized="1";

    line ="<div><table border=0 cellpadding=1 cellspacing=0 bgcolor=#352F7F><tr><td><table width=100% border=0 cellpadding=0 cellspacing=0><tr><td><center><font face='ARIAL' color=#FFFFFF size=2><b>"+ttle+"</b></font></center></td></tr></table><table width=100% border=0 cellpadding=2 cellspacing=0 bgcolor="+fond+"><tr><td><font color=#000000 size=1 face=Verdana>"+msg+"</td></tr></table></td></tr></table></DIV>";

    if (ns)
    {
        skn.document.write(line);
        skn.document.close();
        skn.visibility = "visible";
    }
    else if (ie) 
    {
        document.all("topdeck").innerHTML = line;
        skn.visibility = "visible";
    }
    else if (ns7)
    {
        doc.innerHTML = line
        doc.style.visibility="visible";
    } 
}

function kill() 
{
    initialized="0";
    if (ie || ns) 
    {
        skn.visibility = "hidden";
    }
    else if (ns7)
    {
        doc.style.visibility="hidden";
    }
}

function get_mouse(e)
{
    var x = (ns || ns7) ? e.pageX : event.x+document.body.scrollLeft;
    var y = (ns || ns7) ? e.pageY : event.y+document.body.scrollTop;
    if (initialized == \"1\") {
        if (ns || ie)
        {
            skn.left   = x + 10;
            skn.top    = y - 60;
        }
        else if (ns7)
        {
            doc.style.left   = x + 10;
            doc.style.top    = y - 60;
        }
    }
}

if (ns) document.captureEvents(Event.MOUSEMOVE);

document.onmousemove = get_mouse;

</script>
<table width='100%' border='0' cellpadding='0' cellspacing='0'>
<tr>
<td rowspan='2'>
<img src="$logoicon" alt="LOGO" align=middle>
</td>
<td valign="bottom" align="right">
<font color=#000077>
<h2><img src="$webroot/icons/cabie.gif" width="150" height="90" alt="$product"></h2>
</font>
</td>
</tr>
</table>
<p><b>Mouse over any icon for additional information</b>
<br><hr><br>
<p><b>Key : </b>
<br><br>
<table border=1 cellPadding=15 cellSpacing=1>
<tr>
<td align=center width=10% bgColor=#ffffc8><b>$IDLEMSG"</b></td>
<td align=center width=10% bgColor=#FFCC00><b>Syncing"</b></td>
<td align=center width=10% bgColor=#bed2ff><b>Building"</b></td>
<td align=center width=10% bgColor=#ff9248><b>Overdue"</b></td>
<td align=center width=10% bgColor=#669999><b>Generating Email</b></td>
<td align=center width=10% bgColor=#33cc99><b>Build Untested</b></td>
<td align=center width=10% bgColor=#b9ffb9><b>Build Passed Test</b></td>
<td align=center width=10% bgColor=#FE5066><b>Build Failed</b></td>
<td align=center width=10% bgColor=#dc1973><b>Failed Test</b></td>
</tr>
</table>
<br>
<table border=2 cellSpacing=1>
<tr>
<td align=center width=10%><b>Job # or F</b> - Download Files/View Logs </td>
<td align=center width=10%><b>Job # or L</b> - View 'building' Logs</td>
<td align=center width=10%><b>U</b> - View Job Update</td>
<td align=center width=10%><b>NA</b> - Not Available</td>
</tr>
</table>
<table>
<tr>
<td><i>Jobnames are linked to build instructions</i></td>
</tr>
</table>
<br>
<table frame=box cellSpacing=1 cellPadding=2 bgColor=#cccccc border=1>
<tbody>
<tr bgColor=#aaaaff>
<td rowspan=2 bgColor=#aaaaff><b>    Time    </b></td>
EOF

    #
    # If there were no args just supply the date/time
    #
    if ($HASARGS == 0) {
        $hereandnow = _gen_time_string($itsnow,0);
push @webarray, <<EOF;
</tr><tr>
<td bgColor=#DDDDDD>$hereandnow</td>
</tr>
</table>
EOF
    }

    if ($HASARGS) {
        $sqlquery    = $qs->formatquery(26, $hostname);
        @sqlarray    = $os->run_sql_query("$sqlquery", ":");
        ($desc, $port, $stat) = split(/:/, $sqlarray[0]);
        my $alttext  = "$desc\nStatus: running on port $port";
        my $srvpopup = _do_popup($alttext, "Server Info");

        @statusfile  = jobstate();
        push @webarray, "<td colspan=$numjobs><b>$hostname</b>".
                        "<a href=\"javascript:nothing();\" $srvpopup>".
                        "<img src=\"$infoicon\" border=\"0\" ".
                        "width=\"20\" height=\"20\"".
                        " align=\"middle\"></a>\n".
                        "</td>\n";
    }

push @webarray, <<EOF;
</tr>
<tr>
EOF

    foreach $entry (sort keys %jobhash) {
        $disabled = 0;
        foreach $line (@statusfile) {
            if ($line =~ /\[$entry build job disabled/) {
                $disabled = 1;
                $printline = $line;
                chomp $printline;
            }
        }
        
        #
        # If the build is now disabled
        #
        if (!$disabled) {
            push @webarray, "<td align=\"center\" width=\"100\" ".
                            "bgColor=#DDDDDD align=left>".
                            "<b><a href=$cgibin/jobinfo?$entry?$hostname?".
                            "$port>$entry</a></b></td>\n";
        } else {
            push @webarray, "<td nowrap align=\"center\" width=\"120\" ".
                            "bgColor=#DDDDDD ".
                            "align=left><IMG SRC=\"$disicon\"". 
                            "ALT=\"$printline\"><b><a ".
                            "href=$cgibin/jobinfo?$entry?$hostname?$port>".
                            "$entry</a></b></td>\n";
        }
    }
     
push @webarray, <<EOF;
</tr>
<tr>
EOF

}

sub _inthefuture {

    my $recno = shift;
    my $name  = shift;

    $os->tracefunction((caller(0))[3]);

    if (!defined($timearray[$recno])) {
        return 1;
    }
    if (defined($mainhash{$timearray[$recno]}{$name}{finish})) {
        return 1;
    }

    return 0;

}

sub _get_running_jobs {

    my $jobname   = shift;
    my $server    = shift;
    my $getstring = shift;
    my $jobno;
    my $start;
    my $now;
    my $elapsed;
    my $average;
    my $state;

    my @return;

    my $cmdrtn;

    my $xtraarg;

    $os->tracefunction((caller(0))[3]);

    if ($getstring) {
        $xtraarg = "-v";
    }
    if (defined($server)) {
        $cmdrtn = elapsed("-n", "$jobname", "-r", "-S", "$server", $xtraarg);
    } else {
        $cmdrtn = elapsed("-n", "$jobname", "-r", $xtraarg);
    }
    
    chomp $cmdrtn;

    ($jobno,$start,$now,$elapsed,$average,$state) = split(/,/,$cmdrtn);

    if (defined($state)) {
        $now = $itsnow;
        $elapsed = $now - $start;
        push @stats,  " $jobno,$start,$now,$elapsed,$average,$state";
        push @return, " $jobno,$start,$now,$elapsed,$average,$state";
    }

    return @return;

}

sub _getlastnum {

    my $x;
    my $buildnum;

    $buildnum = $_[0];

    $os->tracefunction((caller(0))[3]);

    $x = rindex($buildnum, "\.");

    return substr($buildnum,$x+1);

}

sub _returnsccs {

    my @Ref = split(/ /, $_[0]);

    $os->tracefunction((caller(0))[3]);

    my @return = split(/;/, $jobHash{$_[0]});
    return $return[0];

}

sub _returncgi {

    my @Ref = split(/ /, $_[0]);

    $os->tracefunction((caller(0))[3]);

    my @return = split(/;/, $jobHash{$_[0]});
    return $return[1];

}


sub _returnjobname {

    my @Ref = split(/ /, $_[0]);

    $os->tracefunction((caller(0))[3]);

    return $Ref[1];

}

sub _returnjobcomment {

    my @Ref = split(/: /, $_[0]);

    $os->tracefunction((caller(0))[3]);

    return $Ref[1];

}

sub _returndepots {

    my $n = @_;
    my $c;
    my @Ref;
    my @MultiRef;

    $os->tracefunction((caller(0))[3]);

    # 8 Is the standard configuration for a single depot...
    if ($n == 8) {
        my @Ref = split(/: /,$_[2]);
        return $Ref[1];
        push @MultiRef, $Ref[1];
    } else {
        for ($c = $n-6; $c > 1; $c--) {
            @Ref = split(/: /,$_[$c]);
            push @MultiRef, $Ref[1];
        }
    }

    return @MultiRef;

}

sub getjoblist {

    my $one = 1;
    my $entry;

    my @ReturnJobArray;

    $os->tracefunction((caller(0))[3]);

    foreach $entry (@JobList) {

        chomp($entry);

        if ($entry =~ /^ Title:/) {
            my ($msg, $title) = split(/:/,$entry);
            push @ReturnJobArray, "$one : $title";
            $one++;
        }

    }

    return @ReturnJobArray;

}

#
# A big mess here...
#
sub _gendisplaystring {

    my @JobInfo = @_;

    my $cc = @JobInfo;

    my $nums = @JobInfo;

    my $entry      = $JobInfo[$nums-1];
    my $JobName    = _returnjobname($JobInfo[1]);
    my $Comment    = _returnjobcomment($JobInfo[$nums-2]);
    my $sccs       = _returnsccs($JobName);
    my $cgiaccess  = _returncgi($JobName);
    my $depot      = $JobInfo[2];
    my @CnArray;

    my $START      = $entry;
    my $FINISH     = $mainhash{$entry}{$JobName}{finish};
    my $CN         = $mainhash{$entry}{$JobName}{jobno};
    my $STATUS     = $mainhash{$entry}{$JobName}{status};
    my $LASTCG     = $mainhash{$entry}{$JobName}{lastcg};

    my @CvsLast;

    my @PrimaryPort  = split(':',$depot);

    my $startstring  = _gen_time_string($START,0);
    my $finishstring = _gen_time_string($FINISH,0);

    my $elapsed      = _format_elapsed_time($FINISH - $START);
    my @p4db;

    # Stuff found by using strict
    my $trunstring;
    my $ttotalstring;
    my $tfinishstring;
    my $CNSHORT;
    my $tlogstring;
    my $c;
    my $d;
    my $byhowlong;
    my @running;
    my $identity;
    my $sccscommand;
    my $refline;
    my $formattedcgi;
    my @promotearray;
    my $promotestring;

    #
    # Javascript strings
    #
    my $popupstring;
    my $commentpop;
    my $infostring;
    my $popstring;
    my $sccspop;

    my $savedcn;

    my $bImrunning = 0;

    #
    # SQL Stuff...
    #
    my $sqlquery;
    my @sqlarray;
    my @queryarray;
    my $sqlline;

    my $promoline;
    my $promopop;

    my $finishimage  = "$runicon";

    chomp $elapsed;

    $os->tracefunction((caller(0))[3]);
        
    #
    # Grab the right sccs icon from buildconf
    #
    my $sccssystem = uc("$sccs");
    my $sccsicon   = $config->{$sccssystem};

    if (($STATUS == 0) || ($STATUS == 1) || ($STATUS == 2) || ($STATUS == 3)) {

        #
        # Setup total and elapsed strings
        #
        $ttotalstring  = "Total: $elapsed";
        $tfinishstring = "Finish: $finishstring";
        
        # 
        # Need to make this generic, but for now this will work...
        #
        if (-d "$Moutdir/$JobName/$CN") {
            my @fullcn  = split(/\./, $CN);
            my $numcn   = @fullcn;
            $tlogstring = "<a href=\"$webserver/$JobName/$CN/\">".
                          "$fullcn[$numcn-1]</a>\n";
        } else {
            $tlogstring = "NA\n";
        }

        #
        # Initialize to nothing
        #
        $trunstring    = "";
        if ($STATUS == 1) {
            $finishimage = "$yellowicon";
        } elsif ($STATUS == 2 ) {
            $finishimage = "$redicon";
        } else {
            $finishimage = "$greenicon";
        }
    } else {

        #
        # Make sure these are blank
        #
        $tlogstring    = "";
        $trunstring    = "";
        
        #
        # Since there's no status we need to see what's up with the job
        #
        @running    = build_status("-n", $JobName);
        $trunstring = "\nProgress: "."$running[0]";

        $bImrunning = 1;

        chomp $trunstring;
        if ($STATUS == 5) {
            if ($running[0] =~ /building retail/ ) {
                $tlogstring = "<a href=\"$cgibin/buildlog?$hostname?$JobName&".
                              "type=0\">L</a>\n";
            } elsif ($running[0] =~ /building debug/ ) {
                $tlogstring = "<a href=\"$cgibin/viewlog.pl?$JobName&".
                              "type=1\">L</a>\n"
            }
        }
        $tfinishstring = "Elapsed: $elapsed\n";
        my $remain     = $mainhash{$entry}{$JobName}{avg} - ($FINISH - $START);
        if ($remain < 0) {
            $byhowlong = _format_elapsed_time(($FINISH - $START) 
                         - $mainhash{$entry}{$JobName}{avg});
            $mainhash{$entry}{$JobName}{status} = 7;
            $ttotalstring  = "Overdue: $byhowlong";
        } else {
            my $remainder  = _format_elapsed_time($remain);
            $ttotalstring  = "TIME LEFT: $remainder";
        }
    }

    chomp $ttotalstring;

    my $cnstring;

    chomp $cnstring;
    @p4db = split(/ /,$PrimaryPort[0]);

    #
    # Create a call to CM broker
    #
    chomp $sccs;
    chomp $depot;
    $sccscommand = $sccs."_identity";

    #
    # Call the broker
    #
    $formattedcgi = $cmbroker->$sccscommand($CN, $depot, $cgiaccess);

    $refline = $formattedcgi;

    $sqlquery = $qs->formatquery(13, $hostname, $JobName, $CN);

    @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

    if (@sqlarray > 0) {
        foreach $sqlline (@sqlarray) {
            @queryarray = split(/;/, $sqlline);
        }
        $promoline = "Promoted at : $queryarray[0]\n".
                     "Promoted by : $queryarray[1]\n".
                     "Comment : $queryarray[2]";

        $promopop  = _do_popup($promoline, "Promotion Info");

        $promotestring = "&nbsp;<a href=\"javascript:nothing();\" $promopop>".
                         "<IMG SRC=\"$promoicon\" width=\"22\" ".
                         "height=\"22\" border=\"0\"></a>";

    } else {
        $promotestring = "";
    }

    #
    # If this job has completed building
    #
    if (($STATUS == 0) || ($STATUS == 1) || ($STATUS == 2) ||($STATUS == 3)) {
    
        $sqlquery = $qs->formatquery(20, $hostname, $JobName, $CN);
    
        @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);
    
        if (@sqlarray > 0) {
            $commentpop  = _do_popup2("$sqlarray[0]", "Build Comment");
            $popupstring = _do_popupstring("$cgibin/gencomment", $hostname, 
                                           $JobName, $CN);
            $infostring = "<a href=$popupstring $commentpop>".
                          "<IMG SRC=\"$openicon\" border=0 width=\"22\" ".
                          "height=\"22\"></a>";
        } else {
            $commentpop  = 
                _do_popup2("Click to enter comments\nfor this build.", 
                           "Enter Comment");
            $popupstring = _do_popupstring("$cgibin/gencomment", 
                                            $hostname, $JobName, $CN);
            $infostring = "<a href=$popupstring $commentpop>".
                "<IMG SRC=\"$closedicon\" border=0 width=\"22\" ".
                "height=\"22\"></a>";
        }
    }

    my $altline = "Jobname : $JobName\n".
                  "Build Server : $hostname\n".
                  "Comment : $Comment".
                  "Job Identifier : $CN\n".
                  "Start : $startstring".
                  "$tfinishstring".
                  "$ttotalstring".
                  "$trunstring";

    $popstring = _do_popup($altline, "Job Info");

    $savedcn   = $cmbroker->$sccscommand($CN,$depot);
    $sccspop   = _do_popup("$savedcn", "CM Info");

    my @zstring;

    if ($bImrunning == 1) {
        @zstring = split(/TIME LEFT: /, $ttotalstring);
    } elsif ($bImrunning == 2) {
        @zstring = split(/Overdue: /, $ttotalstring);
    } else {
       $refline = "<a href=$cgibin/listchanges?srvr=$hostname&".
                  "title=$JobName&startjob=$CN&".
                  "hsrvr=$hostname&htitle=$JobName&sortby=job&".
                  "Query=Generate+List>U</a> "; 
    }

    return $tlogstring.
        $refline.
        "<BR>".
        "<a href=\"javascript:nothing();\" $popstring>".
        "<IMG SRC=\"$finishimage\" border=0 width=\"22\" height=\"22\">".
        "</a>".
        $infostring.
        "&nbsp;<a href=\"javascript:nothing();\" $sccspop>".
        "<IMG SRC=\"$sccsicon\" border=0 width=\"22\" height=\"22\">".
        "</a>".
        $promotestring.
        "<br>".$zstring[1].
        "</td>\n";

}

sub _catch_sig {

    print "args: @_";
    my $signame = shift;
    my $delim = $config->SMTPDELIM;

    #
    # Mail configuration
    #
    my $ms          = $config->SMTP;
    my $admin       = $config->ADMIN;
    my $alternate   = $config->ALTERNATE;

    my %mail;

    $os->tracefunction((caller(0))[3]);

    my $sigtime = sprintf "%s", scalar localtime;

    $mail{Smtp}     = $ms;
    $mail{From}     = "Build Server <$admin>";
    $mail{To}       = "$admin$delim$alternate";
    $mail{Subject}  = "*Urgent - buildserver $hostname unexpected signal ".
                      "caught !*";
    $mail{Message}  = "buildserver $hostname process signaled at $sigtime:\n".
                      "\nCaught SIG$signame\n";

    sendmail %mail;

}

#
# Function for determining whether or not there's enough space for a build
#
sub _freespace {

    my $jobname    = shift;
    my $destintype = shift;

    my $free;
    my $volume;
    my @usage;

    my $delim = $config->SMTPDELIM;

    my $sqlquery;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(15, $jobname, $hostname);
    @usage    = $os->run_sql_query($sqlquery, ";");
    if (!defined($usage[0])) {
        return 0;
    }

    #
    # Unix returns blocks (block size depends on the FS)
    # Win32 returns bytes.  Usage is calculated automatically
    # with each completed job.
    #
    $free   = $os->diskspace($destintype);
    $volume = $os->volume;

    if ($usage[0] > $free) {
        #
        # Mail configuration
        #
        my $ms          = $config->SMTP;
        my $admin       = $config->ADMIN;
        my $alternate   = $config->ALTERNATE;
        my $iscontact   = $config->ISADDR;

        my %mail;

        my $sigtime = sprintf "%s", scalar localtime;

        $mail{Smtp}     = $ms;
        $mail{From}     = "Build Server <$admin>";
        $mail{To}       = "$admin$delim".
                          "$iscontact";
        $mail{Subject}  = "*Urgent - buildserver out of space !*";
        $mail{Message}  = "buildserver $hostname is out of free space:\n".
                          "$volume: $free bytes of unused space\n";

        sendmail %mail;

        return 1;
    } else {
        return 0;
    }

}

#
# Register server in SQL
#
sub _registerserver {

    my $sqlquery;
    my @sqlarray;
    my @sqlsubmit;
    my $myret = 0;
    my $ret;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(66, $hostname);

    @sqlarray = $os->run_sql_query("$sqlquery", ",", 0);

    if (@sqlarray == 0) {

        my $info = $os->sysinfo();

        push @sqlsubmit, "$hostname";
        push @sqlsubmit, "$info";
        push @sqlsubmit, "$Opts{s}";
        push @sqlsubmit, "1";
        push @sqlsubmit, $config->GLOBMAIL;

        if (($ret = $os->run_sql_submit("buildservers", @sqlsubmit)) == 0) {
            $myret = 1;
        }

    } 

    $sqlquery = $qs->formatquery(88, 1, $Opts{s}, $hostname);
    @sqlarray = $os->run_sql_query("$sqlquery", ";", 0);

    return $myret;

}

#
# Separate thread use for polling jobs
#
sub pendingjobs {

    #
    # SQL variables
    #
    my $sqlquery;
    my $lsqlquery;
    my @sqlarray;
    my @lsqlarray;
    my @pollarray;
 
    # 
    # Job variables
    #
    my @properties;
    my $line;
    my $lline;
    my $bCheck = 1;
    my $title;
    my $port;
    my $client;
    my $top;
    my $sccs;

    my $elapsed;
    my @elapsed;
 
    my $command;
    my $pollfile = $config->POLLFILE;
    my $bsr      = $config->BSR;

    my $jobs;

    my @updates;
    my $update;
    my $numupdates;

    #
    # Sleep interval between checking jobs
    #
    my $sleepinterval = 120;
    my $c;

    $os->tracefunction((caller(0))[3]);

    open (PTHREADLOG, ">$dir/logs/pendmon.log");
    PTHREADLOG->autoflush(1);

    $os->writelog("starting pending thread - monitor port: $mport",
                 *PTHREADLOG);

    while (1) {

        @sqlarray = ();

        if (! -f "$bsr/$pollfile") {
            $sqlquery = $qs->formatquery(29, $hostname, 0);
            @sqlarray = $os->run_sql_query($sqlquery, ";");

        } else {

            $os->writelog("opening pollfile $pollfile", *PTHREADLOG);

            open(PPOLLFILE, "<$bsr/$pollfile") || 
                print STDERR "error opening pollfile: $?";

            while(<PPOLLFILE>) {
                chomp;
                if ($_ !~ /^[#\n\t\r\ ]/ && $_ !~ /^$/) {
                    $sqlquery  = $qs->formatquery(27, $hostname, $_, 0);
                    @pollarray = $os->run_sql_query($sqlquery, ";");
                    $sqlarray[$c++] = $pollarray[0];
                }
            }

            close(PPOLLFILE);
            $os->writelog("closed pollfile $pollfile", *PTHREADLOG);

        }
        
        $os->writelog("sqlquery: $sqlquery", *PTHREADLOG);

        foreach $line (@sqlarray) {
         
            if ($line !~ /^$/ ) {

                $os->writelog("line = \"$line\"", *PTHREADLOG);
    
                $bCheck = 1;
         
                @properties = split(/;/, $line);
    
                $title  = $properties[1];
                $port   = $properties[2];
                $client = $properties[3];
                $top    = $properties[4];
                $sccs   = $properties[10];
        
                if ($title !~ /^$/ && $bCheck) {
                    
                    $command = $sccs."_stdoutupdate";
                    # $os->writelog("$command $title, $port, $client, $top, ".
                    #               "$dir", *PTHREADLOG);
                    @updates = $cmbroker->$command($title, $port, $client, 
                                                   $top, $dir);
                    $os->writelog("updates @updates", *PTHREADLOG);
                    $numupdates = @updates;
                    $os->writelog("numupdates $numupdates", *PTHREADLOG);
                    if ($numupdates) {
                        $command = $sccs."_format_pending";
                        my @formatted = $cmbroker->$command(@updates);
                        _pushpending($title, @formatted);
                    } else {
                        #
                        # Make sure any records that exist are purged
                        #
                        my $sqlquery = $qs->formatquery(2, $hostname, $title);
                        $os->run_sql_remove($sqlquery);
                    }
    
                }
            } 

        }
        
        $os->writelog("sleeping for $sleepinterval seconds", *PTHREADLOG);
        sleep($sleepinterval);

    }

    close(PTHREADLOG);

}

#
# Separate thread use for polling jobs
#
sub polljobs {

    #
    # SQL variables
    #
    my $sqlquery;
    my $lsqlquery;
    my @sqlarray;
    my @lsqlarray;
    my @pollarray;
 
    # 
    # Job variables
    #
    my @properties;
    my $line;
    my $lline;
    my $bCheck = 1;
    my $title;
    my $port;
    my $client;
    my $top;
    my $sccs;

    my $elapsed;
    my @elapsed;
 
    my $command;
    
    my $pollfile = $config->POLLFILE;
    my $bsr      = $config->BSR;

    my $jobs;

    my @updates;
    my $update;
    my $numupdates;

    #
    # Sleep interval between checking jobs
    #
    my $sleepinterval = 120;
    my $c;

    my $sport = $os->sport;

    $os->tracefunction((caller(0))[3]);

    open (THREADLOG, ">$dir/logs/mon.log");
    THREADLOG->autoflush(1);

    $os->writelog("starting polling thread - monitor port: $sport",
                  *THREADLOG);

    while (1) {

        #
        # Create a list of locked jobs
        #
        $lsqlquery = $qs->formatquery(22, $hostname, 0);

        @lsqlarray = $os->run_sql_query($lsqlquery, ",");
 
        $os->writelog("$lsqlquery", *THREADLOG);
        
        $c = 0;
        @sqlarray = "";

        if (! -f "$bsr/$pollfile") {

            $os->writelog("no pollfile", *THREADLOG);

            $sqlquery = $qs->formatquery(29, $hostname, 0);
            @sqlarray = $os->run_sql_query($sqlquery, ";");

        } else {

            $os->writelog("opening pollfile $pollfile", *THREADLOG);

            open(POLLFILE, "<$bsr/$pollfile") || 
                print STDERR "error opening pollfile: $?";

            while(<POLLFILE>) {
                chomp;
                if ($_ !~ /^[#\n\t\r\ ]/ && $_ !~ /^$/) {
                    $os->writelog("entry from $pollfile = $_", *THREADLOG);
                    $sqlquery  = $qs->formatquery(27, $hostname, $_, 0);
                    @pollarray = $os->run_sql_query($sqlquery, ";");
                    $sqlarray[$c++] = $pollarray[0];
                }
            }

            close(POLLFILE);
            $os->writelog("closed pollfile $pollfile", *THREADLOG);

        }
        
        foreach $line (@sqlarray) {

            $bCheck = 1;
     
            @properties = split(/;/, $line);
            $os->writelog("$line", *THREADLOG);
            $title  = $properties[1];
            $port   = $properties[2];
            $client = $properties[3];
            $top    = $properties[4];
            $sccs   = $properties[10];
    
            foreach $lline (@lsqlarray) {
                if ($title =~ /^$lline$/) {
                    $bCheck = 0;
                }
            }

            if (!$bCheck) {
                $command = $sccs."_stdoutupdate";
                # $os->writelog("!bcheck $command $title, $port, $client, 
                #                $top, $dir", *THREADLOG);
                @updates = $cmbroker->$command($title, $port, $client, $top, 
                                     $dir);
                $numupdates = @updates;
                $os->writelog("numupdates $numupdates", *THREADLOG);
                if ($numupdates) {
                    $command = $sccs."_format_pending";
                    my @formatted = $cmbroker->$command(@updates);
                    _pushpending($title, @formatted);
                }
            }

            if ($title !~ /^$/ && $bCheck) {

                $os->writelog("checking for updates in \"$title\"", 
                              *THREADLOG);
    
                #
                # Use abstraction for stdoutupdate
                #
                $command = $sccs."_stdoutupdate";
                # $os->writelog("bcheck $command $title, $port, $client, ".
                #               "$top, $dir", *THREADLOG);

                @updates = $cmbroker->$command($title, $port, $client, $top, 
                                     $dir);
            
                $numupdates = @updates;
                $os->writelog("numupdates $numupdates", *THREADLOG);
                if ($numupdates) {
                    _pushpending($title, @updates);
                    _requestbuild($title);
                    _spintilldry($title);
                }

            } else {
                $os->writelog("undefined value found for title, ignoring",
                              *THREADLOG);
            }

        }
        
        $os->writelog("sleeping for $sleepinterval seconds", *THREADLOG);
        sleep($sleepinterval);

    }

    close(THREADLOG);

}

#
# Pendingtest...
#
sub pendingtest {

    $os->tracefunction((caller(0))[3]);

    return "TBD\n";

}

#
# Push pending changes into db
#
sub _pushpending {

    my $title   = shift;
    my @changes = @_;

    my @submit;
    my $list;
    my $string;

    $os->tracefunction((caller(0))[3]);
  
    my $sqlquery = $qs->formatquery(2, $hostname, $title);
    
    $os->run_sql_remove($sqlquery);

    $os->writelog("$sqlquery", *PTHREADLOG);

    foreach $list (@changes) {
        $string .= "$list";
    }

    push @submit, $hostname;
    push @submit, $title;
    push @submit, $string;

    $os->run_sql_submit("pending", @submit);

    $os->writelog("submitted pending files for $title", *PTHREADLOG);

}

#
# On polling thread, wait for build to complete
#
sub _spintilldry {

    my $title = shift;

    #
    # Vars for elapsed info
    #
    my $elapsed;
    my @elapsed;

    #
    # boolean value for waiting
    #
    my $bWait   = 0;
    my $bStart  = 0;
    my $bLocked = 0;
    my $bPending = 0;

    #
    # SQL vars
    #
    $os->writelog("waiting in spintilldry", *THREADLOG);

    my $statequery = $qs->formatquery(74, $title, $hostname, 1);
    my $semquery   = $qs->formatquery(70, $title, $hostname);

    my @sqlarray;
    my $rc;
    my $step;
    my $state;
    my $numsem;

    #
    # Var for sleep interval default to 3 minutes
    #
    my $defsleepinterval = 3*60;
    my $sleepinterval;

    #
    # See if we wait until jobs has completed before moving on
    #
    my $parallel = $config->PARALLEL;

    $os->tracefunction((caller(0))[3]);

    #
    # Sleep long enough to get job started
    #
    $os->writelog("waiting for job to complete", *THREADLOG);
    sleep(30);

    #
    # Query step
    #
    $os->writelog("running $statequery", *THREADLOG);
    @sqlarray = $os->run_sql_query($statequery, ";");
    $rc = @sqlarray;

    #
    # If we got a record and it's not 7 (done)
    # figure out how long to wait
    #
    if ($rc && $sqlarray[0] != 7 && !$parallel) {

        $os->writelog("Run jobs in order", *THREADLOG);

        #
        # Get average time to completion for existing job
        #
        $isinternal = 1;
        $elapsed = elapsed('-n', "$title", '-r');
        $isinternal = 0;

        #
        # Log output for debugging
        #
        $os->writelog("$elapsed", *THREADLOG);

        #
        # Make sure job is running
        #
        if ($elapsed !~ /is not currently running/) {
    
            $bStart = 1;

            while ($bStart) {

                sleep($defsleepinterval);

                @sqlarray = $os->run_sql_query($statequery);

                $step = $sqlarray[0];

                if ($step > 0 && $step < 7) {

                    $os->writelog("running sem $semquery", *THREADLOG);
                    @sqlarray = $os->run_sql_query($semquery, ";");

                    #
                    # If there's more than one entry here there's
                    # a pending lock
                    #
                    $numsem = @sqlarray;

                    #
                    # See if there's a lock pending
                    #
                    if ($numsem == 2) {
                        $bPending = 1;
                    } else {
                        $bPending = 0;
                    }

                    $state = $sqlarray[0];

                    # $os->writelog("state = $state", *THREADLOG);

                    if (!$state) {
                        $bStart = 0;
                        return;
                    } 

                } else {
                    return;
                    if (!$bPending) {
                        $os->writelog("job finished", *THREADLOG);
                        $bStart = 0;
                    }
                }
            }
    
        }

    } 

    return;

}

sub _requestbuild {

    my $title = shift;
    my $conn1;
    my $sport;

    my $sqlquery;
    my @sqlarray;

    my @request = ();

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(57, $hostname);

    @sqlarray = $os->run_sql_query($sqlquery, ";");

    $sport = $sqlarray[0];

    $os->writelog("requesting build of $title on $hostname:$sport", 
        *THREADLOG);

    if (_helo($hostname, $sport)) {
        die "Error: Could not connect\n";
    } else {
         push @request, "build";
         push @request, "-n";
         push @request, "$title";
         
         _rpcrequest($hostname, $sport, @request);
    }

    $os->writelog("build request completed", *THREADLOG);

}

sub _serverup {

    my $ServerName = shift;

    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(71, $ServerName);
    @sqlarray = $os->run_sql_query($sqlquery, ";");

    return $sqlarray[0];

}

sub _rpcrequest {

    my $ServerName = shift;
    my $Port       = shift;
    my $cmd        = shift;

    my $cmdargs = join(/ /, @_);

    my $cli;
    my $request;
    my $resp;

    $os->tracefunction((caller(0))[3]);

    $cli     = RPC::XML::Client->new("http://$ServerName:$Port");
    $request = RPC::XML::request->new($cmd, $cmdargs);
    $resp    = $cli->send_request($request);
    return @{$resp->value};

}

sub _helo {

    my $server = shift;
    my $port   = shift;

    my $cli;
    my $request;
    my $resp;

    $os->tracefunction((caller(0))[3]);

    $cli     = RPC::XML::Client->new("http://$server:$port");
    $request = RPC::XML::request->new("helo", $server);
    $resp    = $cli->send_request($request);

    if ($resp =~ /Connection refused/) {
        return 1;
    } else {
        return 0;
    }

}

#
#  Call this buildserver indirection.  
#
sub _clientcall {

    my $ServerName = shift;
    my @ClientCall = @_;

    my $sqlquery;
    my @sqlarray;

    my @msg;
    my $err;

    $os->tracefunction((caller(0))[3]);

    if (_serverup($ServerName)) {

        $sqlquery = $qs->formatquery(57, $ServerName);
        @sqlarray = $os->run_sql_query($sqlquery, ";"); 

        if (_helo($ServerName, $sqlarray[0])) {
            return "can't establish contact with $ServerName\n";
        } else {
            _logevents("remote call: $ServerName @_", 0);
            @msg = _rpcrequest($ServerName, $sqlarray[0], @ClientCall);
            if (!$isexternal) {
                unshift @msg, "Response from $ServerName:\n";
            }
            return @msg;
        } 

    } else {
        return "Server $ServerName is not running\n";
    }

}

sub _xmlstats {

    my $name  = shift;
    my $job   = shift;
    my $start = shift;
    my $end   = shift;
    my $stat  = shift;

    my $printline;

    my $sstring;
    my $estring;

    $os->tracefunction((caller(0))[3]);

    $sstring = _gen_xml_time_string($start, "start");
    $estring = _gen_xml_time_string($end, "end");

    $printline = "\t<job>\n".
                 "\t\t<host>$hostname</host>\n".
                 "\t\t<jobno>$job</jobno>\n".
                 "\t\t<title>$name</title>\n".
                 "\t\t$sstring\n".
                 "\t\t$estring\n".
                 "\t\t<status>$stat</status>\n".
                 "\t</job>\n";

    return $printline;

}

sub _xmljobinfo {

    my $files    = shift;
    my $comments = shift;
    my $teststat = shift;
    my $running  = shift;
    my $promoted = shift;
    my $limit    = shift;
    my $jobno    = shift;

    my @joblist  = @_;

    my $webroot    = $config->WEBROOT;
    my $jobdir     = $config->JOBDIR;
    my $promotedir = $config->PROMOTION;
    my $stylesheet = $config->STYLESHEET;

    my $promotequery;

    my @promotearray;

    my $title;
    my $server;

    my $entry;
    my $formatted;

    my $sqlquery;
    my @sqlarray;

    my @return;
    my @files;
    my @comments;
    my @tests;
    my @runinfo;

    my $brun = 0;

    $os->tracefunction((caller(0))[3]);

    my @Statusmessage = qw (
        untested
        failedtest
        failed
        passedtest
        syncing
        building
        email
        overdue
    );

    if (defined($stylesheet)) {
        push @return, '<?xml version="1.0" encoding="ISO8859-1" ?>';
        push @return, "\n";
        push @return, "<?xml-stylesheet type=\"text/xsl\" ".
                      "href=\"$stylesheet\"?>";
        push @return, "\n";
    }

    push @return, "<builds>\n";

    my $oldserver = "";
    my $oldjob    = "";
    my $first     = 1;

    if (!$limit && !$jobno) {
        $limit = 10;
    }

    foreach $entry (sort @joblist) {

        ($title, $server) = split(/;/, $entry);

        my $oldtitle;

        if ($oldtitle !~ /$title/) {
            $brun = 0;
        }

        if (!$jobno) {
            $sqlquery = $qs->formatquery(43, $server, $title, $limit);
        } else {
            $sqlquery = $qs->formatquery(42, $server, $title, $jobno);
        }
        
        @sqlarray = $os->run_sql_query($sqlquery, ";");

        foreach my $line (@sqlarray) {
            my $buildurl;
            my $timestring;
            my ($job, $start, $end, $duration, $status, $sccs, $browser, 
                $size, $port) = split(/;/, $line);
            my $bExists = 0;
            $browser =~ s/\&/\&amp;/g;
            if ($sccs =~ /^perforce$/ic) {
                my @j = split(/\./, $job);
                $formatted = sprintf($browser, $j[0])
            } else {
                $formatted = $browser;
            }
            if ($title !~ /^$oldjob$/ic) {
                $oldjob = $title;
                my $sqlq;
                my @sqla;
                my $jobstate;
                my $pending;
                $sqlq = $qs->formatquery(49, $server, $title, 'disabled');
                @sqla = $os->run_sql_query("$sqlq", ",", 1);
                if ($sqla[0] =~ /$title build job disabled/) {
                    $jobstate = "disabled";
                } else {
                    $jobstate = "enabled";
                }
                $sqlq = $qs->formatquery(17, $server, $title);
                @sqla = $os->run_sql_query($sqlq);
                if (@sqla) {
                    $pending = "pending";
                } else {
                    $pending = "";
                }
                if (!$first) {
                    push @return, "    </job>\n";
                }
                push @return, "    <job>\n";
                push @return, "        <title>$title</title>\n";
                push @return, "        <jobstate>$jobstate</jobstate>\n";
                push @return, "        <pending>$pending</pending>\n";
                $first = 0;
            }
            if (!$brun) {
                $oldtitle = $title;
                if ($running) {
                    @runinfo = _xmlrunning($server, $title);
                    @return  = (@return, @runinfo);
                }
                $brun = 1;
            }

            if (-d "$jobdir/$title/$job") {
                $bExists = 1;
                $buildurl = "$webroot/$server/$title/$job";
            }

            #
            # FIXME!
            #
            if (-d "$promotedir/promotion/$title/$job") {
                $bExists = 1;
                $buildurl = "$webroot/promotion/$title/$job";
            }

            push @return, "        <info>\n";
            push @return, "            <num>$job</num>\n";
            push @return, "            <server>$server</server>\n";
            $timestring = _gen_xml_time_string($start, "start");
            push @return, "            $timestring\n";
            $timestring = _gen_xml_time_string($end, "end");
            push @return, "            $timestring\n";
            push @return, "            <duration>$duration</duration>\n";
            push @return, "            <status>$Statusmessage[$status]".
                                       "</status>\n";
            push @return, "            <sccs>$sccs</sccs>\n";
            push @return, "            <browser>$formatted</browser>\n";
            if ($bExists) {
                push @return, "            <buildurl>$buildurl</buildurl>\n";
            } else {
                push @return, "            <buildurl/>\n";
            }
            if ($promoted) {
                my $recs;
                my $query     = $qs->formatquery(13, $server, $title, $job);
                @promotearray = $os->run_sql_query($query,";");
                $recs = @promotearray;
                if ($recs) {
                    my ($at, $by, $comment) = split(/;/, $promotearray[0]);
                    push @return, "            <promotioninfo>\n";
                    push @return, "                <at>$at</at>\n";
                    push @return, "                <by>$by</by>\n";
                    push @return, "                <comment>$comment".
                                                  "</comment>\n";
                    push @return, "            </promotioninfo>\n";
                }
            }
            push @return, "            <size>$size</size>\n";
            push @return, "            <port>$port</port>\n";
            if ($files) {
                @files = _xmlfiles($server, $title, $job);
                @return = (@return, @files);
            }
            if ($comments) {
                @comments = _xmlcomments($server, $title, $job);
                @return = (@return, @comments);
            }
            if ($teststat) {
                @tests = _xmlteststat($server, $title, $job);
                @return = (@return, @tests);
            }
            push @return, "        </info>\n";

        }
    }

    push @return, "    </job>\n";
    push @return, "</builds>\n";

    return @return;

}

sub _xmlrunning {

    my $server = shift;
    my $title  = shift;

    my $cgibin = $config->CGIBIN;

    my @running;
    my @return;

    my $printline;

    my $sccs;
    my $sccsport;

    my $sqlquery;
    my @sqlarray;

    my $port   = $os->port;
    my $itsnow = $os->_get_hosttime();

    my $timestring1;
    my $timestring2;

    my $buildurl;

    $os->tracefunction((caller(0))[3]);

    $isexternal = 1;
    @running = _get_running_jobs($title, $server, 1);
    $isexternal = 0;

    $sqlquery = $qs->formatquery(62, $server, $title);

    @sqlarray = $os->run_sql_query($sqlquery, ";");

    ($sccs, $sccsport) = split(/;/, $sqlarray[0]);

    my ($jobno,$start,$now,$elapsed,$average,$state) = split(/,/, $running[0]);

    #
    # FIXME   "retail" should be the actual job type
    #
    $buildurl = $cgibin."/buildlog\?".$server."\&amp;".$port."\&amp;".
                $title."\&amp;retail";

    if (defined($jobno)) {
        $timestring1 = _gen_xml_time_string($start, "start"); 
        $timestring2 = _gen_xml_time_string($itsnow, "end"); 
        $elapsed = $itsnow - $start;
        $jobno =~ s/ //g;
        $printline = "        <info>\n".
                     "            <num>$jobno</num>\n".
                     "            <server>$server</server>\n".
                     "            $timestring1\n".
                     "            $timestring2\n".
                     "            <duration>$elapsed</duration>\n".
                     "            <average>$average</average>\n".
                     "            <status>building</status>\n".
                     "            <sccs>$sccs</sccs>\n".
                     "            <message>$state</message>\n".
                     "            <browser></browser>\n".
                     "            <buildurl>$buildurl</buildurl>\n".
                     "            <size></size>\n".
                     "            <port>$sccsport</port>\n".
                     "            <teststatus></teststatus>\n".
                     "        </info>\n";
    }

    return $printline;

}

sub _xmlteststat {

    my $server  = shift;
    my $jobname = shift;
    my $job     = shift;

    my $testtitle;
    my $deployserver;
    my $attempt;
    my $attempts;

    my $state;
    my $line;

    my $type;

    my @return;

    my @testplatforms = qw (
        Solaris
        Linux
        Windows
        HPUX
    );

    my @Messages = qw (
        Pending
        Failed
        Running
        Passed
        NA
    );

    my %runstate = (
        waiting    => 1,
        notify     => 1,
        removing   => 1,
        preparing  => 1,
        installing => 1,
        running    => 1,
    );

    my %unitstate = qw (
        unittest,
    );

    my %uistate = qw (
        ui => 1,
    );

    my %completestate = qw (
        sending => 1,
    );

    my @returnstates = qw (
        idle
        running
        passed
        failed
    );

    my $configrecs;
    my $stagerecs;
    my $runrecs;
    my $resultrecs;

    my $sqlqueryconfig;
    my $sqlquerystage;
    my $sqlqueryrun;
    my $sqlqueryresult;
    my $sqlquerynone;

    my @sqlarrayconfig;
    my @sqlarraystage;
    my @sqlarrayrun;
    my @sqlarrayresult;
    my @sqlarraynone;

    my %recordhash;
    my $stateref;

    $os->tracefunction((caller(0))[3]);

    #
    # Get testname for the job and server 
    #
    $sqlqueryconfig = $qs->formatquery(77, $server, $jobname);

    @sqlarrayconfig = $os->run_sql_query($sqlqueryconfig, ";");

    $configrecs = @sqlarrayconfig;

    foreach $line (@sqlarrayconfig) {

        my $rdeployserver;
        my $rattempt = 1;
        my $rstart;
        my $rend;
        my $rmessage;

        my $bDone = 0;

        my $sqlquery;
        my @sqlarray;

        my ($ctitle, $cattempts, $rdeployserver) = split(/;/, $line);

        #
        # Setup type to determine platform type
        #
        if ($ctitle =~ /sol/) {
            $type = 0;
        }

        if ($ctitle =~ /linux/) {
            $type = 1;
        }

        if ($ctitle =~ /win/) {
            $type = 2;
        }

        if ($ctitle =~ /hpux/) {
            $type = 3;
        }

        #
        #  First check to see if test has been completed
        #
        $sqlqueryresult = $qs->formatquery(24, $ctitle, $jobname, $job);

        @sqlarrayresult = $os->run_sql_query($sqlqueryresult, ";");
        
        #
        #  First check to see if test has been completed
        #
        my $tempsql = $qs->formatquery(37, $ctitle, $jobname, $job, 1);

        my @tempsql = $os->run_sql_query($tempsql, ";");

        my $temprecs = @tempsql;

        $resultrecs = @sqlarrayresult;

        #
        # If we found records in the result table then process them
        #
        if ($resultrecs || $temprecs) {

            my $runquery;
            my @runarray;
            my $entry;

            my $counter = 0;

            ($rdeployserver, $rstart, $rend) = split(/;/, $sqlarrayresult[0]);

            $sqlquery = $qs->formatquery(72, $jobname, $server, $job);

            @sqlarray = $os->run_sql_query($sqlquery, ";");

            %{ $recordhash{$ctitle} } = (
 
                Platform   => $testplatforms[$type],
                Servername => $rdeployserver,
                Start      => $rstart,
                End        => $rend,
                Attempt    => $rattempt,
                Message    => "Completed",
                Status     => $Messages[$sqlarray[0]],
    
            );

            $runquery = $qs->formatquery(75, $ctitle, $jobname, $job);
            
            @runarray = $os->run_sql_query("$runquery", ";");

            foreach $entry (@runarray) {
                $recordhash{$ctitle}->{Teststate}[$counter] = "$entry";
                $counter++;
            }

            $bDone = 1;
            
        }

        #
        # Check run table next
        #
        $sqlqueryrun = $qs->formatquery(25, $ctitle, $jobname, $job);

        @sqlarrayrun = $os->run_sql_query($sqlqueryrun, ";");

        $runrecs = @sqlarrayrun;

        #
        # If we found records in the run table then process them
        #
        if ($runrecs && !$bDone) {

            ($rdeployserver, $rstart, $rmessage) = split(/;/, $sqlarrayrun[0]);

            my @compstring = split(/ /, $rmessage);
            my $step;

            my $runquery;
            my @runarray;
            my $entry;

            my $counter = 0;

            #
            # Clean out this reference
            #
            $stateref = [];

            $rend = $os->_get_hosttime();

            $sqlquery = $qs->formatquery(14, $ctitle);

            @sqlarray = $os->run_sql_query($sqlquery, ";");

            if ($runstate{$compstring[0]}) {
                $step = "installing";
            }

            if ($unitstate{$compstring[0]}) {
                $step = "unittest";
            }

            if ($uistate{$compstring[0]}) {
                $step = "uitest";
            }

            if ($completestate{$compstring[0]}) {
                $step = "complete";
            }

            %{ $recordhash{$ctitle} } = (
 
                Platform   => $testplatforms[$type],
                Servername => $rdeployserver,
                Start      => $rstart,
                End        => $rend,
                Attempt    => $sqlarray[0],
                Message    => "$rmessage",
                Status     => $Messages[2],
                State      => $step,
    
            );

            $runquery = $qs->formatquery(75, $ctitle, $jobname, $job);
            
            @runarray = $os->run_sql_query("$runquery", ";");

            foreach $entry (@runarray) {
                $recordhash{$ctitle}->{Teststate}[$counter] = "$entry";
                $counter++;
            }

            $bDone = 1;

        }

        #
        # Check stage table next
        #
        $sqlquerystage = $qs->formatquery(23, $ctitle, $jobname, $job);

        @sqlarraystage = $os->run_sql_query($sqlquerystage, ";");

        $stagerecs = @sqlarraystage;

        #
        # If we found records in the stage table then process them
        #
        if ($stagerecs && !$bDone) {

            ($rdeployserver, $rstart) = split(/;/, $sqlarraystage[0]);

            $rend = $os->_get_hosttime();

            %{ $recordhash{$ctitle} } = (
 
                Platform   => $testplatforms[$type],
                Servername => $rdeployserver,
                Start      => $rend,
                End        => $rend,
                Attempt    => $sqlarray[0],
                Message    => "Pending",
                Status     => $Messages[0],
    
            );

        }

        #
        # There's no records found, set some defaults.
        #
        if (!$bDone) {

             $rstart = $os->_get_hosttime();
             $rend   = $os->_get_hosttime();

             my $realstatus = 4;
             my $realrecs;

             $sqlquery = $qs->formatquery(72, $jobname, $server, $job);

             @sqlarray = $os->run_sql_query($sqlquery, ";");

             $realrecs = @sqlarray;

             if ($realrecs) {
                 $realstatus = $sqlarray[0];
             }

             $sqlquery = $qs->formatquery(81, $ctitle, $jobname, $job);

             @sqlarray = $os->run_sql_query($sqlquery, ";");

             $realrecs = @sqlarray;

             if ($realstatus != 2 && $realrecs) {
                 %{ $recordhash{$ctitle} } = (

                     Platform   => $testplatforms[$type],
                     Servername => $rdeployserver,
                     Start      => $rstart,
                     End        => $rend,
                     Attempt    => $cattempts,
                     Message    => "None",
                     Status     => $Messages[$realstatus],
       
                 );
             }

         }

    }

    #
    # This is the easy part!
    #
    push @return, "            <teststatus>\n";

    foreach my $key (sort keys %recordhash) {
        #
        # Get reference to array if tests are running
        #
        my $reference = $recordhash{$key}->{Teststate};
        my $refent;
        my $timestring;

        push @return, "                <test>\n";
        push @return, "                    <platform>".
                                           "$recordhash{$key}{Platform}".
                                           "</platform>\n";
        push @return, "                    <name>$key</name>\n";
                                           "</platform>\n";
        push @return, "                    <host>".
                                           "$recordhash{$key}{Servername}".
                                           "</host>\n";
        $timestring = _gen_xml_time_string($recordhash{$key}{Start}, "start"); 
        push @return, "                    $timestring\n";
        $timestring = _gen_xml_time_string($recordhash{$key}{End}, "end"); 
        push @return, "                    $timestring\n";
        push @return, "                    <attempts>".
                                           "$recordhash{$key}{Attempt}".
                                           "</attempts>\n";
        push @return, "                    <message>$recordhash{$key}{Message}".
                                           "</message>\n";
        push @return, "                    <step>$recordhash{$key}{Step}".
                                           "</step>\n";
        push @return, "                    <status>$recordhash{$key}{Status}".
                                           "</status>\n";
        foreach $refent (@$reference) {
            my $statedur;
            my ($step, $state, $stateend, $statestart) 
                = split(/;/, $refent);

            if ($stateend < $statestart || $stateend == $statestart) {
                $stateend = $os->_get_hosttime();
            }

            $statedur = $stateend - $statestart;

            push @return, "                    <$step>\n";
            push @return, "                        <resultcode>".
                          "$returnstates[$state]</resultcode>\n";
            push @return, "                        <duration>".
                          "$statedur</duration>\n";
            push @return, "                    </$step>\n";
        }
        push @return, "                </test>\n";
    }

    push @return, "            </teststatus>\n";

    return @return;

}

sub _xmlcomments {

    my $server = shift;
    my $title  = shift;
    my $job    = shift;

    my @return;

    my $sqlquery;
    my @sqlarray;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(20, $server, $title, $job);

    @sqlarray = $os->run_sql_query($sqlquery, ",");

    push @return, "            <comments>\n";

    foreach my $entry (@sqlarray) {
        push @return, "                <message>$entry</message>\n";
    }

    push @return, "            </comments>\n";

    return @return;

}

sub _xmlfiles {

    my $server = shift;
    my $title  = shift;
    my $job    = shift;

    my @return;

    my $sqlquery;
    my @sqlarray;

    my $file;
    my $ver;
    my $change;
    my $who;
    my $sccs;

    $os->tracefunction((caller(0))[3]);

    $sqlquery = $qs->formatquery(61, $server, $title);

    @sqlarray = $os->run_sql_query($sqlquery, ",");

    $sccs = $sqlarray[0];

    $sqlquery = $qs->formatquery(16, $server, $title, $job);

    @sqlarray = $os->run_sql_query($sqlquery, ",");

    push @return, "            <changes>\n";

    foreach my $entry (@sqlarray) {
        ($file, $ver, $change, $who) = split(/;/, $entry);
        push @return, "                <file>\n";
        push @return, "                    <name>$file</name>\n";
        if ($sccs =~ /perforce/ic) {
            push @return, "                    <ver>$ver</ver>\n";
            push @return, "                    <change>$change</change>\n";
        }
        if ($sccs =~ /subversion/ic) {
            push @return, "                    <ver>$ver</ver>\n";
        }
        push @return, "                    <who>$who</who>\n";
        push @return, "                </file>\n";
    }

    push @return, "            </changes>\n";

    return @return;

}

#
# Load list of valid promotion states
#
sub _loadpromotions {

    my %promotestate;

    my $promoconfig = $config->BSR;

    $promoconfig .= "/config/promotestate.conf";

    $os->tracefunction((caller(0))[3]);

    open (PROMOTE, "<$promoconfig" ) || return undef;;

    while (<PROMOTE>) {
        chomp $_;
        if ($_ !~ /^[#\n\t\ ]/) {
            my ($state, $statedir) = split(/\,/, $_);
            $statedir =~ s/^ +//g;
            $promotestate{$state} = $statedir;
        }
    }

    close(PROMOTE);

    return %promotestate;
    
}

#
# Rebuild arg list from client since it's 
# munged by the Msg pkg into a scalar instead
# of an array.
#
sub _parsearray {

    my @in  = @_;
    my @out = ();

    my $entry;
    my $concat;

    my $bstart;

    my $argfound;

    $os->tracefunction((caller(0))[3]);

    foreach $entry (@in) {
        #
        # Look for a hyphen to start an entry
        # 
        if ($entry =~ /^\-/) {
            if (!$bstart) {
                $argfound = 1;
                $bstart   = 1;
            } else {
                $concat =~ s/ +$//g;
                if ($concat !~ /^$/) {
                    push @out, $concat;
                }
                $concat = "";
                $bstart = 0;
            }
            if ($argfound && !$bstart) {
                $bstart   = 1;
                $argfound = 0;
            }
            push @out, $entry;
        } elsif ($entry !~ /^\-/ && $bstart) {
            $concat .= "$entry ";
        } elsif ($entry !~ /^\-/ && !$bstart) {
            push @out, $entry;
            $bstart = 1;
        }

    }

    if ($concat !~ /^$/) {
        $concat =~ s/ +$//g;
        push @out, "$concat";
    }

    @_ = ();

    return @out;

}

#
# Search process tree for children
#
sub _searchProcessHash {

    my $proc = shift;
    my $cmd  = shift;

    my $return;

    $os->tracefunction((caller(0))[3]);

    if (!defined($processtree{$proc})) {
        $foundhash{$proc} = "$cmd";
        return 0;
    } else {
        $return = $processtree{$proc}{ppid};
        if ($return == 1) {
            $return = 0;
        }
    }

    $foundhash{$proc} = "$cmd";

    return $return;

}

sub _get_build_num {

    my $string = shift;

    my @contents;

    @contents = split(/ /, $string);

    return $contents[1];

}
    
sub _get_last_job {

    my $buildname = shift;
    my $recs;

    my $sqlquery;
    my @sqlarray;

    $sqlquery = $qs->formatquery(40, $buildname, $hostname, 1);

    @sqlarray = $os->run_sql_query($sqlquery, ";");

    $recs = @sqlarray;

    if (!$recs) {
        return undef;
    } else {
        #
        # Create an array by splitting up build number
        #
        my (@rollback) = split(/\./, $sqlarray[0]);
    
        #
        # Pop the last element off
        #
        my $lastbuild = pop @rollback;
    
        my ($rollback) = join('.', @rollback);
        return $rollback[0];
    }

}

sub _gen_html_ps {

    my @pstable = @_;

    my @return = ();

    my $entry;

    shift @pstable;

    my $login = $os->getloginname();

    foreach $entry (@pstable) {

        chomp $entry;

        my @line = split(/ +/,$entry);

        my $recs = @line;

        my %tablehash = ();
        my @tabledata = ();

        my $user  = shift @line;
        if ($user =~ /^$/) {
            $user  = shift @line;
        }
        my $pid   = shift @line;
        my $ppid  = shift @line;
        my $cpu   = shift @line;
        my $stime = shift @line;
        my $tty   = shift @line;
        my $time  = shift @line;
        my $cmd   = join(' ',@line);
        my $url;

        if ($login =~ /$user$/ic && $cmd !~ /^ps /) {
            if ($cmd !~ /buildserver\.pl/ic ) {
                $url = "<a href=/adminbroker.php?cmd=kill&server=$hostname".
                       "&pid=$pid&sleep=1>$pid</a>";
            } else {
                $url = $pid;
            }

            %tablehash = (
    
                1 => {
                    content => $user,
                    class   => 'values',
                },
                2 => {
                    content => "$url",
                    class   => 'values',
                },
                3 => {
                    content => $ppid,
                    class   => 'values',
                },
                4 => {
                    content => $cpu,
                    class   => 'values',
                },
                5 => {
                    content => $stime,
                    class   => 'values',
                },
                6 => {
                    content => $tty,
                    class   => 'values',
                },
                7 => {
                    content => "$time",
                    class   => 'values',
                },
                8 => {
                    content => "$cmd",
                    class   => 'values',
                },
                
            );

            @tabledata = tableData(%tablehash);
              
            foreach my $entry (@tabledata) {
                push @return, $entry;
            }

        }

    }

    return @return;

}
